diff --git a/.github/actions/prepare-release-benchmarks/action.yml b/.github/actions/prepare-release-benchmarks/action.yml deleted file mode 100644 index b7bda6f..0000000 --- a/.github/actions/prepare-release-benchmarks/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Prepare release benchmarks -description: Install uncached tools, validate inputs, and inventory full release suites within the caller's shared timeout. - -inputs: - validated-attempt: - description: Workflow attempt that checked the release draft - required: true - -runs: - using: composite - steps: - - name: Require fresh release preflight - shell: bash - env: - VALIDATED_ATTEMPT: ${{ inputs.validated-attempt }} - run: | - set -euo pipefail - if [[ "$VALIDATED_ATTEMPT" != "$GITHUB_RUN_ATTEMPT" ]]; then - echo "::error::Rerun all jobs or dispatch again to revalidate the draft before benchmarking" - exit 1 - fi - - - name: Set up declared tools without caches - uses: ./.github/actions/setup-tools # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... - with: - cache: false - - - name: Inventory full release suites - shell: bash - run: just bench-release-inventory - - - name: Validate benchmark inputs - shell: bash - run: just test-bench-inputs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed6802d..6d98522 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,35 +88,49 @@ jobs: python-version-file: .python-version enable-cache: false - name: Capture successful native validation identities + id: evidence env: GH_TOKEN: ${{ github.token }} run: | - uvx --python 3.15 --from research-repo-tools==0.2.6 python -I - <<'PY' + uvx --python 3.15 \ + --from 'research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e' \ + python -I - <<'PY' import os import sys from pathlib import Path + from research_repo_tools.ci import export_environment from research_repo_tools.cli import main + from research_repo_tools.dependency_provenance import check_development_dependency from research_repo_tools.release_metadata import read_package_info root = Path.cwd() - sys.exit(main([ + if check_development_dependency(root) is not None: + print("Verified development tooling pin; release evidence requires a published supplier.") + sys.exit(0) + status = main([ "release", "evidence", "create", "tooling/release-evidence.toml", "--checkout", str(root), "--commit", os.environ["GITHUB_SHA"], "--version", read_package_info(root).version, "--run-id", os.environ["GITHUB_RUN_ID"], "--attempt", os.environ["GITHUB_RUN_ATTEMPT"], "--output", "target/release-evidence/validation.json", - ])) + ]) + if status == 0: + export_environment(Path(os.environ["GITHUB_OUTPUT"]), ("eligible",), environment={"eligible": "true"}) + sys.exit(status) PY - name: Sign original main validation evidence + if: steps.evidence.outputs.eligible == 'true' id: validation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: target/release-evidence/validation.json create-storage-record: false - name: Retain the signature bundle + if: steps.evidence.outputs.eligible == 'true' env: VALIDATION_BUNDLE: ${{ steps.validation.outputs.bundle-path }} run: cp "$VALIDATION_BUNDLE" target/release-evidence/validation.sigstore.json - name: Upload signed validation evidence + if: steps.evidence.outputs.eligible == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-evidence-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index 125abf9..cabfcca 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -32,7 +32,6 @@ jobs: "files": ["pyproject.toml", "uv.lock"] }, "github_actions": { "files": [ - ".github/actions/prepare-release-benchmarks/action.yml", ".github/actions/setup-tools/action.yml", ".github/workflows/audit.yml", ".github/workflows/benchmarks.yml", diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index c9cf932..60b7556 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -1,8 +1,20 @@ name: Publish to crates.io on: - release: - types: [published] + workflow_dispatch: + inputs: + tag: + description: Published stable tag matching the workflow ref + required: true + type: string + candidate: + description: Independent publication-transfer.json with the canonical attached asset + required: true + type: string + acceptance: + description: SHA-256 of the full acceptance independently reviewed with just release-evidence + required: true + type: string permissions: contents: read @@ -20,8 +32,7 @@ jobs: verify: if: >- github.repository == 'acgetchell/la-stack' && - github.event_name == 'release' && github.event.action == 'published' && - !github.event.release.draft && !github.event.release.prerelease + github.actor == 'acgetchell' && github.triggering_actor == 'acgetchell' runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -29,7 +40,9 @@ jobs: checks: read actions: read outputs: - ci-evidence: ${{ steps.evidence.outputs.identity }} + artifact-id: ${{ steps.approval.outputs.artifact-id }} + artifact-digest: ${{ steps.approval.outputs.artifact-digest }} + validated-attempt: ${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -42,19 +55,25 @@ jobs: with: cache: "false" - - name: Require reviewed release evidence - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: just release-gate "$RELEASE_TAG" - - - name: Verify original signed main CI evidence - id: evidence + - name: Verify the independently reviewed acceptance env: GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_TRANSFER: ${{ inputs.candidate }} + CANDIDATE_ACCEPTANCE: ${{ inputs.acceptance }} run: | set -euo pipefail - just release-evidence --github-output "$GITHUB_OUTPUT" + just release-receive "$RUNNER_TEMP/accepted-candidate" --published + just _release-notes-check "$RELEASE_TAG" + + - name: Freeze the full acceptance before approval + id: approval + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: publication-acceptance-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/accepted-candidate/acceptance.json + retention-days: 30 + if-no-files-found: error publish: needs: verify @@ -78,17 +97,38 @@ jobs: with: cache: "false" + - name: Require this exact approval receipt + env: + VALIDATED_ATTEMPT: ${{ needs.verify.outputs.validated-attempt }} + ARTIFACT_ID: ${{ needs.verify.outputs.artifact-id }} + ARTIFACT_DIGEST: ${{ needs.verify.outputs.artifact-digest }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$VALIDATED_ATTEMPT" == "$GITHUB_RUN_ATTEMPT" ]] || exit 1 + [[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] || exit 1 + [[ "$ARTIFACT_DIGEST" =~ ^[0-9a-f]{64}$ ]] || exit 1 + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID" \ + | jq -e --arg digest "sha256:$ARTIFACT_DIGEST" --arg run "$GITHUB_RUN_ID" \ + '.expired == false and .digest == $digest and (.workflow_run.id | tostring) == $run' >/dev/null + + - name: Download the frozen approval receipt + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ needs.verify.outputs.artifact-id }} + path: ${{ runner.temp }}/approved-acceptance + merge-multiple: true + - name: Recheck evidence and registry absence after deployment approval env: GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - CI_EVIDENCE: ${{ needs.verify.outputs.ci-evidence }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_TRANSFER: ${{ inputs.candidate }} + CANDIDATE_ACCEPTANCE: ${{ inputs.acceptance }} run: | set -euo pipefail - just release-gate "$RELEASE_TAG" - test -n "$CI_EVIDENCE" - printf '%s\n' "$CI_EVIDENCE" > "$RUNNER_TEMP/approved-evidence.json" - just release-evidence --expected "$RUNNER_TEMP/approved-evidence.json" + just release-receive "$RUNNER_TEMP/accepted-candidate" --published --expected "$RUNNER_TEMP/approved-acceptance/acceptance.json" + just _release-notes-check "$RELEASE_TAG" uv run --locked --no-sync research-repo-tools release registry "$RELEASE_TAG" --expect absent - name: Obtain a short-lived crates.io token @@ -106,5 +146,5 @@ jobs: if: ${{ !cancelled() && steps.auth.outcome == 'success' }} env: GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.tag }} run: just release-verify "$RELEASE_TAG" diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index 9b09a79..58c6984 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -1,9 +1,6 @@ name: Release Benchmarks -# Attach full Criterion baselines for maintainer review before publication. -# Release jobs publish durable artifacts, so they intentionally do not restore -# or save dependency caches. - +# Attach the single local acquisition; no job collects benchmark timings. permissions: contents: read @@ -11,7 +8,11 @@ on: workflow_dispatch: inputs: tag: - description: Stable vX.Y.Z tag matching the workflow ref, with a mutable draft release + description: Stable tag matching the workflow ref and mutable draft + required: true + type: string + candidate: + description: Independent owner transfer.json from just release-stage required: true type: string @@ -19,19 +20,18 @@ concurrency: group: release-benchmarks-${{ inputs.tag }} cancel-in-progress: false -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - jobs: - validate-release: - # Draft visibility requires push access. This job executes no repository code. + fetch: + if: >- + github.repository == 'acgetchell/la-stack' && + github.actor == 'acgetchell' && github.triggering_actor == 'acgetchell' + runs-on: ubuntu-latest + timeout-minutes: 15 permissions: + # Draft visibility requires push access. Execute no checkout code. contents: write - runs-on: ubuntu-latest - timeout-minutes: 5 outputs: - target: ${{ steps.target.outputs.target }} + artifact-id: ${{ steps.upload.outputs.artifact-id }} validated-attempt: ${{ github.run_attempt }} steps: - name: Install trusted package runner @@ -41,192 +41,121 @@ jobs: enable-cache: false cache-python: false - - name: Validate draft release target - id: target + - name: Retrieve exact independently selected draft assets env: GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_TRANSFER: ${{ inputs.candidate }} run: | set -euo pipefail - - if [[ "$GITHUB_REF" != "refs/tags/$RELEASE_TAG" ]]; then - echo "::error::Dispatch with --ref $RELEASE_TAG and the matching tag input" - exit 1 - fi - target="$(uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \ - --root "$RUNNER_TEMP" performance release-draft "$GH_REPO" "$RELEASE_TAG" \ - --commit "$GITHUB_SHA" --expected-title "$RELEASE_TAG" | jq -c .)" - echo "target=$target" >> "$GITHUB_OUTPUT" - - measure-baseline: - needs: validate-release - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - suite: [public-default, public-exact] - # Per runner: 2 min checkout + 60 min preparation + 2 min host capture - # + 240 min measurement + 26 min packaging, upload and overhead. - timeout-minutes: 330 - env: - RELEASE_TAG: ${{ inputs.tag }} - BENCH_SUITE: ${{ matrix.suite }} - CRITERION_HOME: ${{ github.workspace }}/target/criterion - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - timeout-minutes: 2 - - - name: Prepare release benchmarks - timeout-minutes: 60 - uses: ./.github/actions/prepare-release-benchmarks # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... - with: - validated-attempt: ${{ needs.validate-release.outputs.validated-attempt }} - - - name: Capture runner host - timeout-minutes: 2 - run: uv run --locked --no-sync research-repo-tools performance host --output target/release-host.json - - - name: Save Criterion suite - id: measurement - timeout-minutes: 240 - run: | - set -euo pipefail - echo "[release-baseline] $BENCH_SUITE started $(date -u +%FT%TZ); budget 240 min" - date +%s > "target/$BENCH_SUITE-started" - just bench-save-baseline "$RELEASE_TAG" "$BENCH_SUITE" - date +%s > "target/$BENCH_SUITE-finished" - - - name: Package Criterion suite - run: | - set -euo pipefail - output="target/release-suites/$BENCH_SUITE" - features=bench - if [[ "$BENCH_SUITE" == public-exact ]]; then - features=bench,exact - fi - uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools performance pack-suite \ - tooling/criterion-saved.toml "$BENCH_SUITE" --saved-sample "$RELEASE_TAG" \ - --output "$output/criterion.tar.gz" --revision "$GITHUB_SHA" \ - --session "$GITHUB_REPOSITORY:$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT" \ - --host target/release-host.json \ - --started-at "$(cat "target/$BENCH_SUITE-started")" \ - --finished-at "$(cat "target/$BENCH_SUITE-finished")" \ - --context "command=just bench-save-baseline $RELEASE_TAG $BENCH_SUITE" \ - --context "features=$features" --context "runner=$RUNNER_NAME" \ - --context "image=$ImageOS" --context "image_version=$ImageVersion" \ - --context "rustc=$(rustc -vV | paste -sd ';' -)" \ - --context "cargo=$(cargo --version)" \ - --context "cargo_lock_sha256=$(sha256sum Cargo.lock | cut -d ' ' -f 1)" - # Transport the native inventory separately for the assembly plan. - # Both archives must match it and the checked-in required case policy. - if [[ "$BENCH_SUITE" == public-default ]]; then - cp target/release-benchmark-inventory.json "$output/inventory.json" - fi - - - name: Upload temporary suite artifact + uvx --python 3.15 \ + --from 'research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e' \ + python -I - <<'PY' + import hashlib, json, os + from pathlib import Path + from research_repo_tools.release_assets import ( + download_release_asset, parse_release_target, publish_release_asset, + revalidate_release_target, + ) + root = Path(os.environ["RUNNER_TEMP"]) + data = json.loads(os.environ["CANDIDATE_TRANSFER"]) + target = parse_release_target(json.dumps(data["target"]).encode("utf-8")) + if (target.repository, target.tag, target.commit, target.expected_title) != ( + "acgetchell/la-stack", os.environ["RELEASE_TAG"], os.environ["GITHUB_SHA"], os.environ["RELEASE_TAG"], + ) or os.environ["GITHUB_REF"] != "refs/tags/" + target.tag: + raise ValueError("dispatch differs from independent release target") + if set(data["assets"]) != {"policy", "publication", "acceptance", "archive"}: + raise ValueError("attachment requires the original four-asset transfer") + def check(): + release = revalidate_release_target(root, target) + for selected in data["assets"].values(): + entries = [item for item in release.assets if item[0] == selected["name"]] + if len(entries) != 1 or type(selected["id"]) is not int or entries[0][1] != selected["id"]: + raise ValueError("original candidate asset ID changed") + if entries[0][3] is not None and entries[0][3] != selected["sha256"]: + raise ValueError("original candidate asset digest changed") + check() + directory = root / "incoming-candidate" + directory.mkdir() + for role, selected in data["assets"].items(): + destination = directory / ("archive.tar.gz" if role == "archive" else role + ".json") + download_release_asset( + root, target.repository, target.tag, selected["name"], destination, + expected_sha256=selected["sha256"], allow_draft=True, + ) + check() + PY + + - name: Upload inert candidate input + id: upload uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: bench-suite-${{ inputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.suite }} - path: target/release-suites/${{ matrix.suite }} + name: candidate-input-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/incoming-candidate/ retention-days: 30 if-no-files-found: error - - name: Suite timing summary - if: ${{ always() }} - env: - SUITE_OUTCOME: ${{ steps.measurement.outcome }} - run: | - set -euo pipefail - elapsed="not started" - if [[ -f "target/$BENCH_SUITE-started" ]]; then - end="$(date +%s)" - if [[ -f "target/$BENCH_SUITE-finished" ]]; then - end="$(cat "target/$BENCH_SUITE-finished")" - fi - elapsed="$((end - $(cat "target/$BENCH_SUITE-started")))" - fi - { - echo "### Release benchmark timing" - echo "" - echo "| Suite | Outcome | Elapsed seconds | Budget minutes |" - echo "| --- | --- | --- | --- |" - echo "| $BENCH_SUITE | $SUITE_OUTCOME | $elapsed | 240 |" - echo "" - echo "An unfinished suite reports elapsed time through this summary." - } | tee -a "$GITHUB_STEP_SUMMARY" - - release-baseline: - needs: [validate-release, measure-baseline] + verify: + needs: fetch runs-on: ubuntu-latest timeout-minutes: 30 - env: - RELEASE_TAG: ${{ inputs.tag }} + permissions: + contents: read + checks: read + actions: read outputs: - release-asset: ${{ steps.package-baseline.outputs.asset }} - artifact-name: ${{ steps.package-baseline.outputs.artifact-name }} - + artifact-id: ${{ steps.upload.outputs.artifact-id }} + artifact-digest: ${{ steps.upload.outputs.artifact-digest }} + validated-attempt: ${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} + fetch-depth: 0 persist-credentials: false - - name: Install trusted package runner - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Set up declared tools without caches + uses: ./.github/actions/setup-tools # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... with: - version: "0.13.7" - enable-cache: false - cache-python: false + cache: false - - name: Download default suite - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: bench-suite-${{ inputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }}-public-default - path: target/release-suites/public-default + - name: Require this retrieval attempt + env: + VALIDATED_ATTEMPT: ${{ needs.fetch.outputs.validated-attempt }} + run: test "$VALIDATED_ATTEMPT" = "$GITHUB_RUN_ATTEMPT" - - name: Download exact suite + - name: Download exact candidate input uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: bench-suite-${{ inputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }}-public-exact - path: target/release-suites/public-exact + artifact-ids: ${{ needs.fetch.outputs.artifact-id }} + path: ${{ runner.temp }}/incoming-candidate + merge-multiple: true - - name: Assemble release Criterion baseline - id: package-baseline - run: | - set -euo pipefail - - asset="la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz" - cp target/release-suites/public-default/inventory.json target/release-benchmark-inventory.json - uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \ - --root "$GITHUB_WORKSPACE" performance assemble-suites tooling/criterion-saved.toml \ - target/release-suites/public-default/criterion.tar.gz \ - target/release-suites/public-exact/criterion.tar.gz \ - --saved-sample "$RELEASE_TAG" --output "$asset" --revision "$GITHUB_SHA" \ - --session "$GITHUB_REPOSITORY:$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT" - echo "asset=$asset" >> "$GITHUB_OUTPUT" - echo "artifact-name=bench-baseline-${RELEASE_TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + - name: Verify original acquisition and full frozen acceptance + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_TRANSFER: ${{ inputs.candidate }} + run: just release-receive "$RUNNER_TEMP/accepted-candidate" --incoming "$RUNNER_TEMP/incoming-candidate" - - name: Upload temporary baseline artifact + - name: Upload verified inert archive + id: upload uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: ${{ steps.package-baseline.outputs.artifact-name }} - path: ${{ steps.package-baseline.outputs.asset }} + name: accepted-candidate-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/accepted-candidate/ retention-days: 30 if-no-files-found: error - publish-baseline: - needs: [validate-release, release-baseline] + attach: + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write - runs-on: ubuntu-latest - timeout-minutes: 10 - + actions: read steps: + # No checkout or repository code runs with attachment credentials. - name: Install trusted package runner uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: @@ -234,36 +163,67 @@ jobs: enable-cache: false cache-python: false - - name: Download release baseline - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ needs.release-baseline.outputs.artifact-name }} - - - name: Attach baseline for review + - name: Require this exact verification attempt and artifact env: + VALIDATED_ATTEMPT: ${{ needs.verify.outputs.validated-attempt }} + ARTIFACT_ID: ${{ needs.verify.outputs.artifact-id }} + ARTIFACT_DIGEST: ${{ needs.verify.outputs.artifact-digest }} GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - RELEASE_TAG: ${{ inputs.tag }} - RELEASE_TARGET: ${{ needs.validate-release.outputs.target }} - RELEASE_ASSET: ${{ needs.release-baseline.outputs.release-asset }} run: | set -euo pipefail + [[ "$VALIDATED_ATTEMPT" == "$GITHUB_RUN_ATTEMPT" ]] || exit 1 + [[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] || exit 1 + [[ "$ARTIFACT_DIGEST" =~ ^[0-9a-f]{64}$ ]] || exit 1 + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID" \ + | jq -e --arg digest "sha256:$ARTIFACT_DIGEST" --arg run "$GITHUB_RUN_ID" \ + '.expired == false and .digest == $digest and (.workflow_run.id | tostring) == $run' >/dev/null + + - name: Download the exact verified artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ needs.verify.outputs.artifact-id }} + path: ${{ runner.temp }}/accepted-candidate + merge-multiple: true - [[ "$RELEASE_ASSET" == "la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz" ]] || exit 1 - printf '%s\n' "$RELEASE_TARGET" > "$RUNNER_TEMP/release-target.json" - uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \ - --root "$RUNNER_TEMP" performance release-upload "$GH_REPO" "$RELEASE_TAG" \ - "$GITHUB_WORKSPACE/$RELEASE_ASSET" --target "$RUNNER_TEMP/release-target.json" - - - name: Release baseline summary + - name: Attach the original accepted archive env: - RELEASE_ASSET: ${{ needs.release-baseline.outputs.release-asset }} + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_TRANSFER: ${{ inputs.candidate }} run: | set -euo pipefail - - { - echo "### Release Benchmark Baseline" - echo "" - echo "Draft release with verified asset: \`$RELEASE_ASSET\`" - echo "Review the draft, then run \`just release-publish \` as the maintainer." - } >> "$GITHUB_STEP_SUMMARY" + uvx --python 3.15 \ + --from 'research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e' \ + python -I - <<'PY' + import hashlib, json, os + from pathlib import Path + from research_repo_tools.release_assets import ( + download_release_asset, parse_release_target, publish_release_asset, + revalidate_release_target, + ) + root = Path(os.environ["RUNNER_TEMP"]) + data = json.loads(os.environ["CANDIDATE_TRANSFER"]) + target = parse_release_target(json.dumps(data["target"]).encode("utf-8")) + if (target.repository, target.tag, target.commit, target.expected_title) != ( + "acgetchell/la-stack", os.environ["RELEASE_TAG"], os.environ["GITHUB_SHA"], os.environ["RELEASE_TAG"], + ) or os.environ["GITHUB_REF"] != "refs/tags/" + target.tag: + raise ValueError("dispatch differs from independent release target") + if set(data["assets"]) != {"policy", "publication", "acceptance", "archive"}: + raise ValueError("attachment requires the original four-asset transfer") + def check(): + release = revalidate_release_target(root, target) + for selected in data["assets"].values(): + entries = [item for item in release.assets if item[0] == selected["name"]] + if len(entries) != 1 or type(selected["id"]) is not int or entries[0][1] != selected["id"]: + raise ValueError("original candidate asset ID changed") + if entries[0][3] is not None and entries[0][3] != selected["sha256"]: + raise ValueError("original candidate asset digest changed") + check() + directory = root / "accepted-candidate" + artifact = directory / f"la-stack-{target.tag}-criterion-baseline.tar.gz" + for role, path in (("acceptance", directory / "acceptance.json"), ("archive", artifact)): + if path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != data["assets"][role]["sha256"]: + raise ValueError("verified artifact differs from independent dispatch assertion") + publish_release_asset(root, target.repository, target.tag, artifact, target=target) + check() + PY diff --git a/Cargo.lock b/Cargo.lock index c76625e..9128e81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" dependencies = [ "find-msvc-tools", "shlex", @@ -1411,18 +1411,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.18+spec-1.1.0" +version = "0.25.19+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec6e78bd70db9420d8d212fa9dc97f116448a1ceff5d00eabb991d407e21396" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" dependencies = [ "indexmap", "toml_datetime", diff --git a/dependency-sources.toml b/dependency-sources.toml new file mode 100644 index 0000000..8cbd869 --- /dev/null +++ b/dependency-sources.toml @@ -0,0 +1,12 @@ +# Shared release validation owns registry and exact development-source checks. +schema = 1 + +[projects.python] +ecosystem = "python" +directory = "." +release-owned = [ "." ] + +[projects.rust] +ecosystem = "cargo" +directory = "." +release-owned = [ ".", "benches/comparison" ] diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 95fd499..ea318fe 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -20,7 +20,7 @@ the commands measure and where their outputs go. - [`vs_linalg` Methodology](#vs_linalg-methodology) - [Exact-Arithmetic Notes](#exact-arithmetic-notes) - [Release Notes](#release-notes) - - [Hosted Release Runtime Budget](#hosted-release-runtime-budget) + - [Hosted Release Verification](#hosted-release-verification) - [Validate The Release Workflow](#validate-the-release-workflow) ## Start Here @@ -40,12 +40,13 @@ Preparation also saves independent release tags and commits for a later | --- | --- | | Working tree versus latest release | `just performance-local` | | Non-exact working tree versus latest release | `just performance-local-non-exact` | -| Two existing tags, full or non-exact | `just performance-measure "$CURRENT_TAG" "$PREVIOUS_TAG" full` | +| Two compatible existing tags, full or non-exact | `just performance-measure "$CURRENT_TAG" "$PREVIOUS_TAG" full` | | Fast saved-baseline loop | `just bench-save-baseline `, then `just bench-compare ` | | Native comparison suite | `just bench-vs-linalg` | | Interval determinant filter | `just bench-interval` | | Certified dot/linear-form filter | `just bench-linear-form` | | Measure and promote a release report | `just performance-release` | +| Resume the original interrupted campaign | `just performance-resume` | | Promote an explicit full scratch run | `just performance-promote` | | Render the retained release report offline | `just performance-doc` | @@ -55,7 +56,10 @@ that scratch pair explicitly and requires a distinct release pair. scratch cannot silently select history. `performance-release` composes fresh preparation, full measurement and promotion; a failed phase stops the recipe. README publication stays in the separate `performance-readme` command so it can -be retried without repeating measurements. +be retried without repeating measurements. Fresh acquisition refuses an existing +checkpoint. Resume keeps the original preparation, completed suite pairs, raw +bytes and journal; it recollects only an incomplete pair. Keep the independent +owner directory printed by the release command outside the checkout. README and plot publication require independent current/baseline tag and full commit assertions. The commands are shown below. Keep those assertions from the @@ -74,7 +78,7 @@ by the automatic benchmark job. | Run `just test-bench-inputs` on Linux | `test-rust-ci` runs the complete all-feature workspace unit and integration suite, including `vs_linalg_inputs`, `exact_bench_config`, `column_basis`, `rational_dot`, `interval_solve`, and `polynomial_bench_inputs`. | | Build the exact harness for timing | `bench-compile` builds the workspace benchmark targets and exact-feature targets with warnings denied. | | Execute the full exact harness and its fixture checks | `test-bench-exact` runs every exact case in Criterion's `--test` mode without timing samples. | -| Save or download a main baseline, sample a PR, and report cross-runner regressions | The manual workflow collects one exploratory run. Tagged release acquisition and its input gate remain in [Releasing](RELEASING.md#6-run-benchmarks-and-review-the-draft). | +| Save or download a main baseline, sample a PR, and report cross-runner regressions | The manual workflow collects one exploratory run. Accepted release evidence is attached through [Releasing](RELEASING.md#6-attach-accepted-benchmarks-and-review-the-draft). | The three native build status checks still cover benchmark validation. The optional [Benchmarks workflow](https://github.com/acgetchell/la-stack/actions/workflows/benchmarks.yml) @@ -106,30 +110,42 @@ for diagnostics. ### Release acquisition policy -The `public-release-and-lu-peers-v3` policy extends #301 with column bases, +The `public-release-and-paired-lu-peers-v4` policy extends #301 with column bases, interval division/solves, rational dot products, and polynomial determinants. The full la-stack selection has 736 default-feature and 667 exact-feature cases. Local `performance-release`, `performance-local`, and full `performance-measure` measure all 1,403 cases at the current revision under one captured harness. -The baseline measures the 1,254 shared cases: 646 default-feature and 608 -exact-feature cases. Column bases, interval division/solves, rational dot products, -and polynomial determinants were introduced in v0.5.2, so their 149 cases have -current timings only. Reports leave their baseline cells empty; they establish -no release-to-release speedup. The independent catalog declares this boundary -under `[baseline]`; revisit it when the supported baseline floor includes these APIs. -Only the baseline phase also measures the 16 native nalgebra/faer LU -factorization-plus-one-solve cases at D=2, 3, 4, 5, 8, 16, 32, and 64. -The matching eight current la-stack timings already belong to the full suite. +Baselines before v0.5.2 measure 1,254 shared library cases: 646 default-feature +and 608 exact-feature cases. Column bases, interval division/solves, rational dot +products and polynomial determinants were introduced in v0.5.2. Their 149 cases +are required on each supported revision and unavailable on older revisions. +The independent catalog's `[availability]` table and the schema-6 suite bounds +declare this boundary. Resolution uses the original source receipt before +overlaying the current harness; a supported compile failure remains an error. +Both phases also measure the 16 native nalgebra/faer LU factorization-plus-one-solve +cases at D=2, 3, 4, 5, 8, 16, 32 and 64. Each phase's eight matching la-stack +cases and 16 peers share one `lu-solve` acquisition unit. The README uses +current-phase peer/current-library ratios; baseline peer observations remain +attributed to their original phase. Peers and la-stack originate in one local acquisition environment; hosted results never supply the denominator of a local peer ratio. -Hosted release acquisition and the default `bench-save-baseline` measure only -la-stack. Discovery, saved-byte verification, and local measurement declarations +Hosted release jobs reuse the original accepted local acquisition without +measuring. The optional `bench-save-baseline` diagnostic measures only la-stack. Discovery, saved-byte verification, and local measurement declarations have separate completeness requirements. `tooling/performance-policy.toml` contains the independent required case catalog; native `--list` discovery retains the observed inventory and rejects missing required cases. The installed engine rejects duplicate semantic IDs and incomplete raw samples. +The full plan uses one sequential trial; README and LU plots explicitly select +trial 1. Baseline selectors include README because historical compilers read it. +Current source includes README only during rustdoc; current native gates verify +that both debug and release dependency files exclude it before timing. Full +comparison with an older current revision that still compiles README is rejected. +The non-exact diagnostic conservatively declares README on both phases. +All source/harness pins and configurations must be settled before release +acquisition; only the precise declared outputs may change afterward. + | Public operation family | Measurement owner and feature configuration | | --- | --- | | Angles: vector, checked slice, construction followed by angle | `angle`, default, vetted dense/sparse/endpoint fixtures at D=3–6; excludes the control kernel | @@ -158,12 +174,21 @@ parameter combination. Broad peer operations, Gaussian and hand-written reference timings, determinant implementation diagnostics and exponent-spread studies remain opt-in. Use `bench-vs-linalg`, `bench-exact`, or the non-exact diagnostic workflow explicitly. -Neither routine release path acquires them. The README retains its existing +Routine release acquisition excludes them. The README retains its existing eight-dimension LU graph; no additional performance page or graph is produced. -The local driver prints the case count and nominal timing budget before samples -are acquired. The current plan has 2,673 measurements and a nominal 356.4-minute -warmup-plus-measurement budget, before compilation, gates, and analysis. +The [v0.5.3 workload review](dev/benchmark-workload-review.md) maps required, +optional and excluded measurements to caller costs, numerical regimes and +oracles. Astra accepted its no-removal conclusion and recommended a paired LU +suite with current-phase peer comparisons for the availability migration. +The paired-suite migration implements that decision within the current schema-6 +schedule; historical acquisitions retain their original policy and bytes. + +The shared reporter prints the discovered case count and nominal timing budget +before samples are acquired. With a pre-v0.5.2 baseline, the plan has 2,689 +measurements and a nominal 358.5-minute warmup-plus-measurement budget. A baseline +at v0.5.2 or later requires 2,838 measurements and 378.4 nominal minutes. +Both budgets exclude compilation, gates and analysis. `performance-release` records preparation, measurement and promotion elapsed seconds and exit status in ignored `target/bench-reports/full/runtime.txt`, including failures. The timer starts after Just's prerequisite checks. Use `time just performance-release` @@ -174,11 +199,33 @@ a nominal 189.6-minute budget; that calculation supplies no observed elapsed time. Expanded public coverage means removing peers alone establishes no overall speedup. No end-to-end acquisition has been timed for the expanded policy. -Full comparisons require both revisions to implement every selected API. A -baseline predating these additions fails discovery; it cannot supply polynomial, -rational-dot, column-basis or interval-solve ratios. Hosted tagged acquisition -still records absolute costs. Existing retained reports remain historical and -do not satisfy the expanded inventory; changing this plan does not rewrite them. +`performance-release` uses the installed shared progress reporter. Its optional +arguments select output mode and heartbeat seconds: + +```bash +just performance-release human 300 +just performance-release json 60 +``` + +These are alternative acquisition commands, not report retries. The default is +human output with five-minute heartbeats; `none` disables shared status. Stage +transitions are immediate. JSON records on stderr use schema +`research-repo-tools/acquisition-progress/v1` and may be interleaved with native +diagnostics. Captured logs need no TTY. Counts advance only after phase validation; +partial output files do not count. ETA stays unknown for unseen work, and neither +an estimate nor the nominal budget is a timeout. Scheduled progress reports +trial, suite and phase state, with eight nominal seconds per case. +The current development pin fixes the gate-failure event defect recorded in the +review. Preserve the command's nonzero exit and runtime record when diagnosing +failure. + +Retry a failed report promotion with `just performance-promote`, or README +publication with `just performance-readme`. Neither reacquires samples. + +Each suite compares revisions only when both implement its API. Earlier baselines +retain explicit unavailable cells for new APIs; supported discovery failures +remain errors. Existing retained reports remain historical and do not satisfy +the expanded inventory; changing this plan does not rewrite them. **`public_api`** (`benches/public_api.rs`) owns construction, access, mutation, symmetry, direct determinant certificates, interval determinant queries, and @@ -355,8 +402,9 @@ timing. The full gate covers `vs_linalg_inputs`, `exact_bench_config`, `column_basis`, `rational_dot`, `interval_solve` (including its rational oracle), and `polynomial_bench_inputs`. The non-exact diagnostic retains its original comparison and exact-configuration gate. -Baseline measurements include nalgebra and faer; current measurements reuse -those baseline-phase peers and measure la-stack and the exact suite. Diagnostic +The full plan includes nalgebra and faer in both LU-suite phases and selects +current-phase peers for README ratios. The non-exact diagnostic retains its +baseline-phase peer selection. Diagnostic cases remain in the complete samples even when omitted from selected report rows. Every case requires 100 samples, mean and median estimates, 95% confidence @@ -409,9 +457,8 @@ rows and does not establish common-harness release eligibility. Use `performance-local` for that gate. Native Criterion reports remain useful for individual benchmarks. -The hosted release workflow discovers the full inventory, measures each suite -once, saves its requested baseline name and verifies exact saved copies before -packaging. The installed verifier is also available directly: +For an optional saved-baseline diagnostic, the installed verifier checks the +full discovered inventory and exact saved copies directly: ```bash just bench-release-check "$CURRENT_TAG" target/criterion @@ -440,7 +487,7 @@ just performance-readme "$CURRENT_TAG" "$CURRENT_COMMIT" "$PREVIOUS_TAG" "$PREVI ``` If the preparation record is missing or invalid, publication stops with guidance. -Recapture the reviewed inputs or supply all four explicit assertions; neither +Restore the original owner-held preparation or supply all four reviewed assertions; neither publication path repeats measurements or promotes a different report. The canonical README view is LU solve, median, logarithmic time axis, declared @@ -477,7 +524,8 @@ views consume the non-exact scratch pair and use separate There is no implicit input fallback. Plots preserve scientific coordinates, nanoseconds, the selected statistic -and original baseline-phase peers. They render evidence without measuring or +and each series' originating phase. Full views use current-phase peers; the +non-exact diagnostic retains baseline-phase peers. They render evidence without measuring or probing the current host. Independent identity and measurement-policy checks still apply. README publication additionally requires live source freshness. @@ -852,7 +900,7 @@ or `det_exact_f64_result (vs det_exact_f64)`. Those used the historical lossy `*_exact_f64` baseline. New comparisons require matching benchmark identifiers; the pre-v0.4.3 name substitutions have been removed. -The default `public-release-and-lu-peers-v3` scope includes canonical exact-arithmetic +The default `public-release-and-paired-lu-peers-v4` scope includes canonical exact-arithmetic groups because their inputs and execution order are fixed across revisions. Historical baselines created before the `exact_random_corpus_d*` names were introduced do not have comparable full-corpus rows, so those rows appear once both sides of a @@ -872,103 +920,58 @@ just bench-save-baseline just bench-save-last ``` -The durable published baseline is the GitHub Release artifact created by -`.github/workflows/release-benchmarks.yml`. That workflow runs the benchmark-input -correctness gate before timing or packaging the artifact. The committed release -comparison is `docs/performance.md`, created by `just performance-release`. - -Follow [Releasing](RELEASING.md#5-create-the-draft-github-release): create the -tagged stable release as a draft, dispatch the workflow with -`--ref "$TAG" -f tag="$TAG"`, and let the workflow upload and verify the archive. -Review the draft, then run `just release-publish "$TAG"` as the maintainer to -start the separate crates.io workflow. Dispatch requires a stable `vX.Y.Z` tag and exactly one -mutable draft with that tag as its title. The producer checks out the resolved -tag commit only when it matches the workflow's own commit. The dispatch ref -must be that same tag, which keeps execution in the tag's cache scope; the -publisher rechecks the commit and captured release ID. -Missing releases, prereleases, and published releases are rejected before -benchmarking. Publication makes the attached evidence immutable. - -### Hosted Release Runtime Budget - -The producer runs default and exact public workloads in parallel on separate -`ubuntu-latest` runners. Each job allows 330 minutes: 2 for checkout, 60 for -preparation, 2 for host capture, 240 for measurement, and 26 for packaging, upload and -overhead. Each preparation runs full discovery and independent input validation, -reserving 35 minutes for uncached tools, two 10-minute discovery deadlines and -5 minutes for the input gate. Both runners compile the selected targets before -timing; neither restores dependency caches. Each measurement step has at least -50% headroom beyond its planning estimate. These remain capacity estimates until -the expanded hosted policy completes. - -| Selection | Cases | Nominal warmup + measurement | Planning estimate at 12 s/case | Step limit | -| --- | --- | --- | --- | --- | -| `public-default` | 736 | 98.1 min | 147.2 min | 240 min; 330 min job | -| `public-exact` | 667 | 88.9 min | 133.4 min | 240 min; 330 min job | -| Hosted aggregate work | 1,403 | 187.1 min | 280.6 min | Concurrent jobs, then 30 min for assembly | -| Local total: shared baseline cases, all current cases and 16 peers | 2,673 | 356.4 min | 534.6 min | Separate per-phase deadlines | - -These estimates are not measured runtimes or upper bounds. Criterion can extend -measurement for expensive iterations. Sampling remains unchanged: 100 samples, -100,000 resamples, 95% confidence, 3-second warmup and 5-second measurement -target. Native filters select public workloads without reducing their sampling. - -Hosted wall time follows the slower suite plus preparation and archive assembly, -subject to runner availability. Parallel execution increases setup work because -each runner installs and validates its own environment. The archive records each -suite's runner, CPU, toolchain, lockfile hash, command, source commit, workflow -attempt and measurement timestamps under `release-suites/`. These are -absolute timings from two hosts; do not treat cross-suite values as a common-host -comparison. Local baseline/current and peer comparisons still use one captured -environment. - -The workflow logs discovered counts and budgets before timing. Its final summary -records each feature selection's outcome, elapsed seconds and budget, including -failed steps. A runner-level termination can prevent the summary. For comparable -before/after runtime evidence, retain command, source/harness identity, toolchain, -dependencies, runner/host and cache state, plus setup and acquisition elapsed -times. Do not infer a speedup from the number of removed peer rows. Historical -completed v0.5.1/v0.5.0 evidence remains unchanged and requires no rerun. - -`just bench-release-inventory` requires a fresh Criterion directory and obtains -IDs from each selected native binary's `--list` output. It checks the independent -la-stack catalog; hosted completeness deliberately does not require peer rows. -`just bench-release-check ` requires every discovered ID, mean/median -estimates with 95% intervals and 100 finite positive samples. Each named -baseline's four raw JSON files must match its fresh `new` measurement, including -the native sanitized storage paths. Missing public cases, failed writes, stale -baselines and malformed measurements stop publication. For one suite, use -`just bench-release-check-suite ` with the runner's output directory. -The shared `pack-suite` command performs that validation before -packaging, so the workflow needs no additional verification pass. It retains -original host metadata captured before timing and digests of every raw file. -Assembly uses `assemble-suites` with the checked-in policy and separately -transported native inventory. It requires both suites from the same commit and -repository/run/attempt, matching complete contracts, disjoint portable paths, -and unchanged raw bytes. Only successful assembly emits the final archive and -Actions artifact output. The separate publisher attaches that archive -as `la-stack-$TAG-criterion-baseline.tar.gz` to the draft, verifies its uploaded -state, size, and SHA-256 digest, and leaves the draft ready for review. - -Starting with v0.5.2, the shared archive contains raw Criterion paths directly -at its root, `criterion-inventory.json`, and original manifests at -`release-suites/0.json` (default) and `release-suites/1.json` (exact). It has no -outer `criterion/` directory. Historical assets retain that wrapper and their -original bytes. Extract a new archive into an absent directory, then verify it: - -```bash -uv run --locked research-repo-tools performance extract "$ASSET" target/release-criterion -just bench-release-check "$TAG" target/release-criterion -``` - -The verification command also needs the independently discovered -`target/release-benchmark-inventory.json`; the archived inventory alone does not -replace the checked-in required scientific coverage. Original per-suite host -records remain authoritative when reading the archive on another machine. +The durable baseline is derived from the accepted Step 1 local acquisition. +The committed comparison is `docs/performance.md`. Follow +[Releasing](RELEASING.md#step-2-publish-after-the-pr-is-merged) to verify the +final clean commit, create a draft, stage the independent records, and dispatch +attachment and publication. Both dispatches use the exact tag and owner-generated +selection JSON; downloaded files cannot authenticate their own assertions. + +### Hosted Release Verification + +`release-benchmarks.yml` performs retrieval, read-only candidate/native +verification and attachment in separate jobs. Only the no-checkout retrieval and +attachment jobs have repository write permission. The checkout verifier has +read permission and uses an isolated pinned supplier runtime. It never repeats +measurement, discovery, native gates or package builds. + +The independent selection fixes the original draft ID, acquisition and +publication digests, and exact transport asset IDs/hashes. The verifier checks +the original archive and full frozen final acceptance. Its immutable artifact +passes by exact ID/digest and current attempt to the attachment job. That job +rechecks the original draft and input assertions and attaches identical accepted +bytes as `la-stack-$TAG-criterion-baseline.tar.gz`. + +Before publishing, the owner records the canonical attached archive ID/hash in +a separate publication selection. The crates.io dispatch also requires the +owner-reviewed full acceptance SHA-256. Verification freezes that exact receipt +before protected deployment approval and revalidates it afterward. A newer +complete native attempt requires an explicit new review and dispatch; the +original immutable archive and release assets remain unchanged. Changed +transport assets, canonical archives, tags, titles, generated notes or immutable +release state stop publication. + +There is no hosted measurement budget. Local sampling remains 100 samples, +100,000 resamples, 95% confidence, 3-second warmup and 5-second measurement. +With v0.5.2-or-later baseline availability, 2,838 case observations imply 378.4 +minutes of nominal warmup/measurement plus setup; this is a planning estimate, +not measured runtime or an upper bound. Resume retains completed suite pairs +inside the single scheduled trial without discarding original provenance. + +The candidate archive retains `evidence/run.json`, its evidence envelope, +current Criterion files and `reuse.json`. Final acceptance distinguishes the +original measured source from the final release commit. Original local host, +session and timestamps remain authoritative. + +Previously published suite archives remain readable through shared extraction +and saved-sample verification. Their original layouts, inventories, host records +and bytes stay unchanged. The optional `bench-release-inventory`, +`bench-release-check` and `bench-release-check-suite` recipes remain diagnostic +tools; they do not create or authenticate candidate acceptance. ### Validate The Release Workflow -Run the applicable local gates for workflow changes: +For workflow changes, run the applicable gates: ```bash just lint-config @@ -977,25 +980,14 @@ just markdown-ci just doc-check ``` -The Python suite executes the workflow's shell with the pinned installed CLI -and a simulated GitHub provider to prove captured-target handoff, commit-change -rejection and preservation of the draft for review. Generic provider failure matrices belong -upstream. Archive fixtures retain scientific coverage and exact raw bytes. - -Every hosted dispatch requires a real release draft and attaches verified -evidence for review. Use the -[release sequence](RELEASING.md#6-run-benchmarks-and-review-the-draft) for a -planned release. Record the successful run URL and both elapsed suite times -when validating a budget change. The estimates above still require this -representative hosted run; local tests and archive fixtures do not establish -GitHub-runner runtime or upload success. - -The temporary artifact is named -`bench-baseline-$TAG--` and retained for 30 days. -Retry a failed producer by rerunning all jobs or dispatching again, so the -draft is checked in the same attempt before setup or measurement starts. -Rerunning only failed publisher jobs reuses the successful producer's artifact. -An existing draft asset is reused only when its bytes match; conflicting assets -require inspection and manual removal while the release is still a draft. -Published releases are always rejected, and `--clobber` is never used. See -[failed-run recovery](RELEASING.md#recovering-a-failed-run) before retrying. +Consumer fixtures exercise actual trusted workflow blocks, native Just argument +transport, exact asset/attempt selection, capture and staging retries, full +candidate verification and unchanged raw bytes. Synthetic samples establish +workflow behavior, not performance. Native platform CI and a real maintainer +dispatch remain separate operational evidence. + +Every dispatch uses a real release and authenticated owner assertions. No test +uploads artifacts or obtains registry credentials. Follow +[failed-run recovery](RELEASING.md#recovering-a-failed-run): rerun complete +hosted attempts with the same selection, preserve local checkpoints and owner +records, and never overwrite conflicting assets or silently select new evidence. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 1817c01..003e710 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -1,14 +1,16 @@ # Releasing la-stack -Prepare each `vX.Y.Z` release in a dedicated PR. After that PR is merged, -create the annotated tag and a draft GitHub release. Start `Release Benchmarks` -to attach the durable baseline, then review and publish the draft with -`just release-publish`. The published release starts `Publish to crates.io`; -approve its `crates-io` deployment to permit the Cargo upload. - -The local benchmarks used to prepare the release PR do not require a draft. -The draft is needed later as the upload destination for the benchmark archive -produced on GitHub Actions. +Prepare each `vX.Y.Z` release in a dedicated PR. Acquire the comparison once +locally during Step 1, keeping original assertions outside the checkout. +After merge and native CI, verify that acquisition against the final clean +commit before tagging. Create a draft, stage accepted records, and dispatch +`Release Benchmarks` to verify and attach the original archive. Publish the +reviewed draft, explicitly dispatch `Publish to crates.io` with the independent +publication selection, then approve its `crates-io` deployment. + +Both workflows require the maintainer's authenticated dispatch against the exact +release tag. Neither measures benchmarks. Development supplier pins support +rehearsal and ordinary checks; actual release work requires published tooling. The changelog is generated for the target tag before the tag exists, so the release process does not require a temporary local tag. @@ -19,7 +21,8 @@ same retained bundle produces no file changes. Rerunning `update-version` on a later UTC date intentionally advances the declared release date. Crossing midnight alone requires no update: the `declared` date policy allows preparation and tagging on different UTC days when citation and changelog dates agree. -Repeated `performance-release` runs can produce different measurements. +Fresh acquisition requires a new checkpoint. Use `performance-resume` for an +interrupted campaign; completed suite pairs retain their original bytes. With unchanged history and configuration, repeating `changelog-release` reuses the declared citation date and preserves the generated contents, including when a rerun happens after UTC midnight. @@ -34,53 +37,60 @@ including when a rerun happens after UTC midnight. ## Reusable CI evidence -The publishing process uses the installed `research-repo-tools==0.2.6` -release evidence commands. `tooling/release-evidence.toml` declares the repository, +The publishing process uses the pinned shared tooling's release evidence +commands. `tooling/release-evidence.toml` declares the repository, package, workflow, native jobs/steps and signer policy. The guarantee is `signed-source`; it does not promise promotion of an identical Cargo archive. -Every successful protected `main` push produces signed evidence after all three -native `just ci` jobs and the default/exact Cargo packaging job succeed. +Protected `main` pushes produce signed evidence only with published tooling, +after all three native `just ci` jobs and the default/exact Cargo packaging job +succeed. A development Git pin passes ordinary validation but cannot produce +release evidence. CI verifies its development identity and skips signing and +uploading the release-evidence artifact; ordinary validation still runs. The `release-evidence` job signs their exact commit, run, attempt, job IDs and package identity, then retains the manifest and Sigstore bundle in an immutable Actions artifact for 30 days. -`just release-evidence` verifies that evidence for the current checkout. It -requires one unambiguous successful `push` run on `main` from this repository's -active CI workflow, every required job and step from that same attempt, and a -nonexpired artifact retrieved by immutable ID. The shared downloader checks its -authoritative ZIP size and SHA-256 digest and exact file inventory. GitHub CLI -then verifies the signed manifest's workflow, source commit, `main` ref, hosted -runner, push event, run and attempt. The manifest must match the live job and -package identities. Pending, -failed, ambiguous or expired evidence stops publication. PR, manual dispatch, -fork, local and partial-rerun results cannot replace it. - -Local verification prints a success message; pass `--output ` to retain -the full approval selection. In Actions, -`just release-evidence --github-output "$GITHUB_OUTPUT"` uses the installed -transactional output writer to append `identity` and `commit` only after -verification succeeds. The destination must -already exist; export errors fail the command. Existing output records survive, -and normal or quiet Just execution transports the same JSON without capturing -stdout. The shared command owns JSON validation and command-file transport. -The Just recipe launches the exact published package with `uvx` outside the -observed checkout, using native Python argument transport on Windows. -Omitting `--expected` selects fresh evidence; a supplied empty or malformed -selection file is an error. Existing selection files can only be reused with -identical bytes. The installed downloader bounds archives at 32 MiB and the -signed manifest at 1 MiB; repository, digest and signature checks still apply. - -The publishing workflow reuses these completed checks and passes the run/attempt -and artifact ID/digest through `steps.evidence.outputs.identity` and -`needs.verify.outputs.ci-evidence` to `CI_EVIDENCE`. After deployment approval, -it requires a nonempty value, writes it to `RUNNER_TEMP/approved-evidence.json`, -and revalidates it with `just release-evidence --expected ` before token -acquisition. A new run, attempt, artifact, manifest or policy cannot replace -the approved selection. -It also retains the independent release metadata, exact-commit SARIF, -durable Criterion asset, immutable GitHub release and registry-absence gates. -Release benchmarks remain a separate scientific measurement workflow; CI -evidence does not replace their full sampling or durable archive. +`just release-evidence ` verifies original acquisition, +the precise publisher receipt, final clean tree and signed native evidence. +It requires one successful `push` run on `main`, every required job and step +from that attempt, and a nonexpired immutable artifact. The shared verifier +checks the ZIP digest, exact inventory, signature, workflow, source commit, +hosted runner and package identity. PR, manual-dispatch, fork and partial-rerun +results cannot replace main-push evidence. + +Retain the owner directory printed by `performance-release` and an independent +backup under exclusive maintainer control. Its default location is +`~/.local/state/la-stack/acquisitions/`. An ignored +checkout directory is not an authentication boundary. Original raw evidence, +preparation, acquisition assertions, publication receipt and their digests +are immutable. + +Final acceptance lives under `/final///`. +The default selection is `reviewed`. Repeating verification revalidates the +full acceptance, including the original native run, attempt and artifact. +To explicitly review a newer complete CI attempt, use a new name such as +`just release-evidence "$OWNER" retry-2` and pass that same selection to +archive, staging and publication recipes before GitHub publication. The command +prints the full acceptance file's SHA-256. After GitHub publication, renew only +that acceptance and supply its reviewed digest to a fresh Cargo dispatch; +the original release assets and publication selection remain unchanged. +Existing records are never replaced. + +These recipes launch the pinned supplier through `uvx` outside the observed +checkout. Step 2 does not measure, discover, rebuild or mutate Git. It writes +only requested receipts outside the checkout. `release-native-evidence` +remains a focused native-CI diagnostic; it does not establish candidate reuse. + +The independently retained `transfer.json` selects the exact draft release +and four inert asset IDs/hashes. After attachment, `publication-transfer.json` +also selects the canonical archive ID and original hash. These operator-supplied +dispatch inputs authenticate the assertions; hashes from received candidate +files cannot authenticate those same files. The Cargo dispatch also requires +the independently reviewed full acceptance digest. Verification freezes those +exact bytes in an immutable Actions artifact before approval; the upload job +revalidates that receipt after approval. Both jobs check the original asset +selection and immutable release state; generated notes and registry absence +remain publication gates. This reuses validation of the source. Cargo still packages the clean release commit when uploading, so it does not promise byte-for-byte promotion of the @@ -93,6 +103,18 @@ constraints used by the installed verifier. See GitHub's ## Prepare the environment +Replace any development tooling pin with its supplying published release and +reconcile the manifest, lock and isolated runtime pins as described in +[Shared capability adoption](../tooling/README.md#shared-capability-adoption). +Before release acquisition, run `just release-dependencies`. This shared gate +rejects Git sources even when their version label matches a published package; +release evidence and publication repeat the check. Retain development results +as rehearsal evidence and revalidate the final published-pin inputs. +The root `dependency-sources.toml` declares the Python tooling project and the +Cargo workspace's release-owned packages. Keep the private comparison member +explicit; other dependency packages require verified registry artifacts. Include +this policy with manifests and locks in source and benchmark evidence. + Set the target tag once: ```bash @@ -232,16 +254,46 @@ The selected tags and commits are printed before measurement. A failed fetch, release lookup or preparation stops the recipe before benchmarking. README publication remains a separate, retryable step below. +The optional progress arguments are `just performance-release human 300` +(the default) or, for captured JSON status, `just performance-release json 60`. +These commands acquire measurements. Shared status reports the active native +stage, validated counts, elapsed time and an ETA only when its observations +support one. Five-minute heartbeats do not limit native command runtime. +The [benchmark guide](BENCHMARKING.md#release-acquisition-policy) explains +unknown budgets/estimates and the pinned supplier's gate-failure event limitation. +The shell runtime record still retains the final duration and exit status. +After acquisition succeeds, retry report promotion with `just performance-promote` +and README publication separately; neither retry performs measurements. + For preparation alone, `just performance-prepare` uses local tags without changing refs or running benchmarks. Add `--fetch-tags --allow-git-mutations` to refresh tags explicitly. This separate command remains useful for inspecting -inputs or recovering a missing preparation record before retrying README publication. +inputs before acquisition. Restore a lost original preparation from its +independent owner record; never derive replacement assertions from a candidate. The shared preparation record also binds the package version and measurement declaration. If you have an old adapter-format `release-inputs.json`, remove that ignored scratch file once and prepare again. It is deliberately not converted from measurement evidence. Moved refs or changed package/declaration inputs require a new preparation before measurement or README publication. -Each `performance-release` invocation prepares fresh inputs before measuring. +Fresh acquisition refuses an existing checkpoint before preparation. +Resume explicitly reuses the original preparation and checkpoint: + +```bash +just performance-resume +``` + +The default checkpoint is `target/bench-reports/full/checkpoint`; pass another +path as the first resume argument when needed. One sequential trial retains +completed suite pairs. An interrupted pair is recollected in full; completed +pairs, original sessions and the append-only journal remain intact. Changed +inputs, incompatible declarations or ambiguous work stop recovery. + +Before report promotion, the adapter freezes the independent owner record. +After a capture failure, preserve scratch and retry +`just performance-capture "$OWNER"`. If raw bytes did not reach that record, +explicitly use `just performance-capture "$OWNER" --retained-scratch` while +you still control the original acquisition. Then retry promotion and README +rendering directly. Never treat incoming candidate files as owner assertions. No shell variables or exports are needed. `performance-prepare` does not set `PREVIOUS_TAG`, `CURRENT_COMMIT`, or `PREVIOUS_COMMIT` in your shell; those @@ -253,13 +305,23 @@ new run and index changes under `docs/performance-runs/`. Its current/baseline tags and commits must match the selected assertions. Both labels must be v0.5.0 or newer. -Evidence retains both phases, mean and median estimates, 100 raw samples per -case and 95% intervals. APIs introduced in v0.5.2 retain current timings only; -their baseline cells are unavailable because v0.5.1 cannot perform those operations. -Shared APIs retain comparisons between both revisions. Current declarations record acquisition policy as well -as source/harness provenance. Future runs support offline rendering after -scratch cleanup; `just performance-doc` explicitly selects retained history. -`just performance-promote` explicitly selects the full scratch pair. +Evidence retains every available phase, mean and median estimates, 100 raw +samples per case and 95% intervals. Availability uses the original revision: +APIs introduced in v0.5.2 are unavailable earlier and paired on v0.5.2-or-later +baselines. No supported missing case is treated as unavailable. All 1,403 +library cases and 16 LU peer cases on each phase remain required. + +The full schema-6 plan declares one sequential trial. Baseline inputs include +README because historical releases compile it. Current source includes README +only during rustdoc; current gates inspect debug and release dependency files +and reject a compiler dependency on README before timing. A historical current +revision that still compiles README cannot use this full candidate plan. +Use explicit non-exact diagnostics when appropriate. + +Complete dependency pins, source, harness, declarations and release metadata +before acquisition. Later changes outside the precise output contract invalidate +candidate reuse. Reports and README select retained trial 1; no extra trial or +counterbalancing claim is implied. Historical artifacts, including the v0.5.0/v0.4.6 comparison, remain frozen with their original bytes and links. That older run lacks the new acquisition @@ -290,12 +352,11 @@ release tag is pushed. Until then, preview the SVG directly under The four explicit tag/commit arguments remain supported as described in the [benchmark guide](BENCHMARKING.md#update-the-readme-nalgebrafaer-table). -If the saved inputs are missing or invalid, the command stops with preparation -guidance. Recover the reviewed inputs with `just performance-prepare` or supply -the four explicit arguments, then retry `performance-readme`. Do not rerun -`performance-release` solely to retry README publication: it repeats preparation -and measurements. A README failure leaves the retained measurements available for -another publication attempt. +If saved inputs are missing or invalid, restore the original preparation from +the independent owner record or supply the four independently reviewed +assertions, then retry `performance-readme`. Do not start a new acquisition +solely to retry publication. A README failure leaves retained measurements +available for another publication attempt. See [Benchmarking](BENCHMARKING.md) for explicit scratch versus retained plot inputs, named metric/statistic/axis views, output ownership and methodology. @@ -370,43 +431,42 @@ git switch main git pull --ff-only ``` -### 2. Create and verify the annotated tag +### 2. Verify the final release commit + +Set `OWNER` to the independent directory printed during Step 1: ```bash -just tag "$TAG" -git --no-pager tag -l --format='%(contents)' "$TAG" +OWNER="/path/to/independent/acquisition" +just release-evidence "$OWNER" +just release-archive "$OWNER" ``` -`just tag` builds the annotation from the matching active or archived changelog -section. For a changelog larger than 125 kB, the annotation points to that -section instead of embedding it. +Require native Linux, macOS and Windows CI, default/exact Cargo packaging, +signed main-push evidence and exact-commit SARIF checks. The candidate verifier +also checks that the final tree differs from the measured candidate only through +the accepted publication contract. It preserves the original measured revision +separately from the final commit. Do not tag until verification succeeds. + +If main-push evidence needs renewal, rerun every job in the original run and +explicitly review a new selection. Manual CI dispatch cannot supply release +evidence. Pending, failed, cancelled or expired evidence blocks publication. -### 3. Push the tag +### 3. Create and verify the annotated tag ```bash -git push origin "$TAG" +just tag "$TAG" +git --no-pager tag -l --format='%(contents)' "$TAG" ``` -### 4. Verify the release commit +`just tag` builds the annotation from the matching active or archived changelog +section. For a changelog larger than 125 kB, it links that section. + +### 4. Push the tag ```bash -just release-evidence +git push origin "$TAG" ``` -Require successful native `CI` builds on Linux, macOS and Windows, plus Clippy -and repository-rule SARIF checks, for this exact commit before proceeding. -These builds include the declared Rust 1.99.1 baseline, default/exact doctests, -all-feature tests, interval and linear-form regressions, examples and benchmark -compilation. The publishing policy in `pyproject.toml` binds all five check names -to the GitHub Actions App. CodeRabbit and human review remain protected-PR -requirements; CodeRabbit does not run on the merged `main` commit. Missing, -pending, cancelled or failed native checks block publication. The evidence -command also checks the original main CI packaging and signing jobs. If no -eligible push run exists, stop and correct the event-delivery problem before -selecting the release commit. A manually dispatched CI run cannot produce -release evidence. A failed or expired main-push run requires a complete rerun -of that original run, with fresh evidence from the new attempt. - ### 5. Create the draft GitHub release Create an unpublished release where the workflow can attach its benchmark archive: @@ -423,97 +483,79 @@ Leave it as a stable draft: publishing now would make the release immutable before the benchmark archive can be attached. GitHub recommends this [draft-first sequence for immutable releases](https://docs.github.com/en/code-security/concepts/supply-chain-security/immutable-releases). -### 6. Run benchmarks and review the draft +### 6. Attach accepted benchmarks and review the draft + +Stage the four inert records: + +```bash +just release-stage "$OWNER" +``` -Start the workflow from the release tag, passing the same tag as its input: +This revalidates acceptance, freezes the draft ID before any upload, and prints +the independent `transfer.json` path. A retry uses that same draft and identical +bytes. Dispatch from the exact tag using the printed path: ```bash -gh workflow run release-benchmarks.yml --ref "$TAG" -f tag="$TAG" +TRANSFER="$OWNER/final/$(git rev-parse HEAD)/reviewed/transfer.json" +gh workflow run release-benchmarks.yml --ref "$TAG" -f tag="$TAG" -F candidate=@"$TRANSFER" ``` -Once GitHub accepts the command, you can close the terminal and leave the -workflow running. Check its status later on the -[Release Benchmarks Actions page](https://github.com/acgetchell/la-stack/actions/workflows/release-benchmarks.yml). - -The workflow runs from the tagged version of its definition, so the tag must -include this release workflow. Using the tag as the workflow ref keeps execution -in that tag's cache scope. Dispatching from `main` or a different tag is rejected. - -The workflow automatically: - -1. Runs the full public la-stack workload selection for the tagged commit; - local acquisition separately owns the README's 16 peer LU-and-solve cases. -2. Uploads the archive to the draft and verifies its name, state, size, and - SHA-256 digest. -3. Leaves the draft and verified archive ready for maintainer review. - -A benchmark, upload, or verification failure leaves the release as a draft. -Follow [Recovering a failed run](#recovering-a-failed-run) to retry. - -Before benchmarking, the workflow requires -exactly one mutable, non-prerelease draft whose tag and title match the stable -`vX.Y.Z` input, and resolves the existing tag to a commit. It benchmarks that -commit only if it matches the workflow's own commit, then rechecks the captured -release ID and tag commit before attaching -the archive. Do not edit the draft, move the tag, -or publish manually while the workflow is running. - -The privileged jobs install the exact published `research-repo-tools==0.2.6` -package on Python 3.15 without repository checkout. Preflight emits versioned -target JSON with the captured ID, commit and title assertion. Its trusted job -output passes directly to the publisher, separately from the producer's inert -archive. The shared writer verifies downloaded asset bytes as well as provider -metadata. It leaves publication to the authenticated maintainer. - -The measurement jobs capture their own host metadata before timing and use the -shared `pack-suite` command to verify and archive their full samples. Assembly -uses `assemble-suites` with the checked-in policy, separately transported native -inventory, tagged commit and repository/run/attempt identity. It emits the final -asset only after validating both suites, raw-file digests and original provenance. -These suite commands use the published `research-repo-tools==0.2.6` through -`uvx --python 3.15`, matching the standard development dependency pin. -New v0.5.2 archives use the shared layout described in -[Benchmarking](BENCHMARKING.md#hosted-release-runtime-budget); historical assets -keep their original layout and bytes. - -Runs for the same tag are serialized without cancelling an active run. +The no-checkout retrieval job checks original release/asset identities and +downloads the selected bytes. A separate read-only checkout job verifies +acquisition, final binding and full frozen native acceptance. The no-checkout +attachment job receives an immutable artifact by exact ID, digest and current +attempt, rechecks the draft and original assertions, then attaches +`la-stack-$TAG-criterion-baseline.tar.gz`. No job measures or builds. + +The archive retains original evidence, current Criterion files and a separate +final-release binding. Local ownership, measured revisions, hosts, sessions and +timestamps remain unchanged; acceptance does not relabel their source. + +Review the draft title, generated notes and archive. The workflow leaves it +unpublished. Do not edit, replace or publish it while a job is running. Runs for +the same tag serialize without cancellation. Privileged jobs use isolated +pinned tooling and execute no checkout code. ### 7. Publish the reviewed draft and approve Cargo publication -After the workflow completes, open the repository's -[Releases page](https://github.com/acgetchell/la-stack/releases). Review the -draft, generated notes and `la-stack-$TAG-criterion-baseline.tar.gz` asset. -From the clean tagged checkout, using your human GitHub CLI session, run: +From the clean tagged checkout and your human GitHub CLI session: + +```bash +just release-publish "$TAG" "$OWNER" +``` + +This verifies acceptance and remote notes, freezes the canonical attached archive +ID/hash in `publication-transfer.json`, then publishes the reviewed draft +through the shared gate. It authorizes GitHub Release publication. The release +must become immutable before Cargo publication. + +Supply that independently retained selection to the second explicit dispatch: ```bash -just release-publish "$TAG" +just release-evidence "$OWNER" +# Copy the printed SHA-256 of the full acceptance after reviewing its selection. +ACCEPTANCE_SHA="" +PUBLICATION="$OWNER/final/$(git rev-parse HEAD)/reviewed/publication-transfer.json" +gh workflow run publish-crates.yml --ref "$TAG" -f tag="$TAG" \ + -F candidate=@"$PUBLICATION" -f acceptance="$ACCEPTANCE_SHA" gh run list --workflow publish-crates.yml --limit 5 ``` -This command authorizes GitHub Release publication. The shared gate checks the -clean checkout, stable tag/package metadata, remote tag commit, protected default -branch ancestry, successful exact-commit checks and required baseline asset. -The local recipe also verifies signed main CI evidence and requires the release -title and body to match the tag and generated notes. Human authentication is essential: publishing with a workflow's -`GITHUB_TOKEN` does not trigger the separate `release: published` workflow. - -The publishing workflow checks out `github.sha`, rechecks the published event's -repository, release ID, tag/ref/commit and evidence, then verifies the original -signed main CI evidence without repeating the native checks or packaging builds. -It also reads the captured release ID to require GitHub's immutable state. -After it succeeds, approve the `crates-io` environment deployment in GitHub. -The upload job rechecks the same selected run, attempt and artifact ID/digest, -the release gate, and proven absence of the exact registry version after approval, -then obtains a short-lived OIDC token. Within the publishing workflow, only that -job has `id-token: write`; it has no repository write permission or dependency caches. -Uploads for `la-stack` are serialized without cancellation. - -The original main CI job builds the default and `exact` packages without upload -credentials. After approval, Cargo repackages the same clean, revalidated commit -with `--locked --no-verify`, so build scripts do not run with the crates.io token. -The dry-run archive is not transferred between jobs, and byte-identical archives -are not asserted. This follows [Cargo's native publication model](https://doc.rust-lang.org/cargo/commands/cargo-publish.html). -No benchmark workspace member is published. +The first job checks immutable release state, canonical and transport asset +IDs/hashes, original archive acceptance, final clean commit and generated notes. +It verifies current native evidence against the separately reviewed full +acceptance digest and freezes the exact receipt in an immutable Actions artifact. +After it succeeds, approve the `crates-io` deployment in GitHub. The upload job +retrieves that artifact by exact ID, digest and current run/attempt, repeats +verification against its frozen receipt, proves the registry version +absent, then requests an OIDC token. Only that job has `id-token: write`; it +has no repository write permission or dependency caches. Uploads serialize +without cancellation. + +Original main CI built default and `exact` packages without upload credentials. +Cargo repackages the clean commit with `--locked --no-verify`, so build scripts +do not run with the crates.io token. This is signed source validation, not +byte-identical Cargo archive promotion. No benchmark workspace member is published. ### 8. Verify registry availability and downstream resolution @@ -546,78 +588,52 @@ commit, benchmark run, publishing run, registry URL/checksum and consumer result in the current release's tracking issue or merged release PR. Keep any release tracking issue open until publication and consumer verification succeed. -The benchmark producers have read-only repository permissions and restore no -dependency caches, including tool binaries. Each invokes `setup-tools` with -`cache: false`, installs declared managed Rust and Cargo tools through the locked -shared package, and exports verified paths. The short preflight job receives `contents: write` -for draft visibility; the separate asset writer receives it to attach the archive. -Neither privileged benchmark job checks out or executes repository code. - -Default-feature and exact-feature workloads run on separate runners in parallel. -Each job allows 330 minutes, including 60 minutes for uncached tool installation, -compilation, inventory and input validation, and 240 minutes for measurement. -The 1,403 cases include column bases, interval division/solves, rational dot -products, and polynomial determinants. Both suites retain full release sampling. Inventory -and raw-data validation must succeed for both suites. A separate 30-minute job -checks their commit/attempt identities and disjoint files, validates the complete -dataset, and packages one archive with provenance for each runner. Inspect the -suite timing summaries when a run fails. -See the [hosted runtime budget](BENCHMARKING.md#hosted-release-runtime-budget) -for measured history, capacity estimates, and headroom, and -[workflow validation](BENCHMARKING.md#validate-the-release-workflow) for local -regression checks and hosted evidence requirements. +Hosted jobs restore no dependency caches. The checkout verifier has read-only +repository permissions. Draft retrieval and attachment require write access +but execute no checkout code. See +[hosted verification](BENCHMARKING.md#hosted-release-verification) and +[workflow validation](BENCHMARKING.md#validate-the-release-workflow) for the +exact handoff, local regressions and separate hosted evidence requirements. ### Recovering a failed run -For reusable CI evidence, rerun **all** jobs in the original main-push CI run. -Failed-jobs-only reruns omit producers or validators from the new attempt and -are rejected. A complete rerun creates a new signed artifact; it never changes -the earlier artifact. If CI was rerun while deployment approval was pending, -the pinned old attempt is stale: restart the publishing workflow's verification -and obtain approval for the newly selected evidence. Do not bypass expiration -or substitute manual-dispatch results. Local builds are useful diagnostics but -cannot renew hosted attestations. - -The following recovery rules apply separately to `Release Benchmarks`. -If preflight, either producer, or archive assembly failed, rerun all jobs with -`gh run rerun ` or dispatch again. Each producer requires a preflight from the current attempt; -rerunning failed benchmark jobs alone stops before tool setup or measurement -because the earlier draft check is stale. - -A rerun uses the original workflow at the release tag; it does not pick up a -workflow fix merged later. If the tagged workflow needs repair, keep the release -as a draft and have the maintainer review the corrected release commit and tag -before dispatching again. Never move a published tag. - -If the producer succeeded and only the publisher failed, preserve the existing -30-day Actions artifact and rerun failed jobs: +Resume local acquisition with `just performance-resume`; keep the original +checkpoint, preparation and owner record. Retry capture, report promotion and +README publication separately after acquisition completes. Source/harness drift +requires a new reviewed acquisition, not repaired receipts. + +For native evidence, rerun **all** jobs in the original main-push run. Partial +attempts and manual CI dispatch remain ineligible. Accept a new complete native +attempt through an explicit new review selection: ```bash -gh run rerun --failed +just release-evidence "$OWNER" renewed-ci ``` -The publisher reuses an already uploaded draft asset only if its state, size, -and digest match the downloaded archive. It never overwrites or deletes assets. -A conflicting or incomplete upload stops the run; inspect the draft and remove -only that conflicting asset manually before retrying. An expired temporary -artifact requires a new full run. - -A full rerun or new dispatch creates a distinct Actions artifact named -`bench-baseline-$TAG--`. Fresh measurements may differ -from an earlier uploaded draft asset, so resolve any conflict before retrying -publication. If the release is already published, every rerun fails closed -without changing it, including when an earlier publication succeeded but its -response was lost. Verify the public release in that case. Existing immutable -releases with missing assets cannot be repaired by this workflow. - -If the crates.io upload fails or Cargo times out, inspect the original run and -exact version before retrying. The workflow checks visibility even after an -upload failure but preserves the failed job result. A visible version makes a -rerun stop before OIDC/upload; use `just release-verify` and the clean consumer -check to finish verification. A registry transport/authentication error does not -prove absence. After a visibility timeout, investigate the accepted-upload state -before authorizing a rerun for the unchanged tag; never blindly retry an upload. -Corrections to published source require a new version. Never move a published tag. +Before GitHub publication, use the new selection for archive, staging and +attachment. After GitHub publication, keep the original immutable archive, +transport assets and `publication-transfer.json`. Review the new acceptance and +dispatch `publish-crates.yml` again with its full SHA-256 as `acceptance` and +the original publication selection as `candidate`. This renews CI evidence +without reacquiring timings or replacing release assets. A changed native +attempt after deployment approval requires a fresh reviewed dispatch. + +For hosted recovery, rerun all jobs or dispatch again with the same independent +selection. Inert artifacts are selected by exact ID and attempt and retained for +30 days. Failed-jobs-only attachment reruns cannot reuse another attempt's +verification. These retries perform no timings. + +Upload retries reuse the original draft ID and identical bytes. Conflicting +or incomplete assets stop without overwriting or deletion. Inspect failures +before manual correction. A replacement draft needs a new explicit review +selection. Never move a published tag or replace independent digests with values +from received files. + +A rerun uses the tagged workflow. A later workflow fix requires a reviewed new +candidate. If Cargo reports an ambiguous timeout, inspect exact registry +visibility before retrying; a present version must not be uploaded again. +`just release-verify "$TAG"` checks the release and registry with bounded +retries. Keep protected deployment approval and all pre-upload checks enabled. ### 9. Remove the merged release branch diff --git a/docs/code_organization.md b/docs/code_organization.md index efc9f61..c038af2 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -165,7 +165,7 @@ inherit the root workspace lints and share the lockfile and target directory. The [Benchmarking guide](BENCHMARKING.md) owns benchmark commands, methodology, baselines, output locations, and report promotion. The [Tooling guide](../tooling/README.md) owns the performance declaration inventory, consumer-test scope, and recipe ownership. -The pinned published `research-repo-tools` dependency +The pinned `research-repo-tools` dependency owns changelog generation, normalization, minor-series archiving, note lookup, and tag preparation through its CLI. Consumer identity and dependency-body policy stay in `[tool.research-repo-tools.changelog]` in `pyproject.toml`, with @@ -192,9 +192,9 @@ and review policy belong in [Security Checks](../SECURITY.md#security-checks). The shared package also owns managed tool installation, verification, and update implementation. `.python-version`, `rust-toolchain.toml`, and `pyproject.toml` own consumer declarations; `tests/tooling/test_toolchain_integration.py` checks -the exact published pin, registry lock, installed version and distribution hashes, +the exact supplier source, lock and installed identity, alongside actual recipe sequencing and native managed execution. The -`.github/actions/setup-tools/action.yml` composite synchronizes the locked PyPI +`.github/actions/setup-tools/action.yml` composite synchronizes the locked package and exports verified paths for CI. Release callers disable its caches. `.github/actions-allowlist.json` owns approved action identities for local @@ -232,24 +232,33 @@ Hosted Dependabot approvals use the pinned shared GitHub workflow; the [rollout guide](dev/MANAGING_CHANGES.md#dependabot-approval-rollout) owns settings and deployment verification. -`tooling/criterion-inventory.toml` declares hosted la-stack-only list commands and +`tooling/criterion-inventory.toml` declares optional saved-baseline list commands and independent public API coverage. `tooling/criterion-saved.toml` declares complete saved-sample requirements. Shared discovery and saved-sample APIs own inventory parsing and exact raw-file verification. The shared package also owns suite-scoped verification, provenance, packaging and complete archive assembly. -The release workflow retains the feature matrix, native commands, job budgets -and artifact handoff; its regression and archive tests live in -`tests/tooling/test_release_baseline.py`. +Legacy archive and saved-sample regressions live in +`tests/tooling/test_release_baseline.py`; new release archives reuse accepted local evidence. There is no repository-specific suite verifier or archive merge implementation. `tooling/performance.just` declares native Cargo discovery, correctness gates and measurements. `tooling/measurements/{full,non-exact}.toml` declares the -minimum release, complete sampling, independent required cases, original -baseline-phase peers, sources, harness, probes, environment and compatibility. +minimum release, complete sampling, independent required cases, availability, +phase-owned peers, sources, harness, probes, environment and compatibility. `tooling/performance-policy.toml` remains the independent scientific catalog -used by consumer acceptance tests and the local budget display. Full acquisition -adds only 16 peer LU-and-solve cases in the baseline phase; the non-exact -diagnostic retains broader peers. Runtime commands consume the shared schemas. +used by consumer acceptance tests. The schema-6 full plan schedules one sequential trial and groups paired LU +measurements, shared default/exact workloads and four API-introduction suites. +It includes 16 LU peer cases on each phase and selects current-phase peers for +README ratios; the non-exact diagnostic retains broader baseline peers. +Runtime commands consume the shared schemas. +The root `dependency-sources.toml` declares Python/Cargo release ownership for +shared dependency validation and is included in benchmark harness inputs. +The [workload review](dev/benchmark-workload-review.md) records the scientific +selection argument, supplier contract findings and remaining adoption evidence; +it does not replace the independent case catalog. +The [candidate review](dev/benchmark-candidate-review.md) records the independent +release-ordering and trust review for resume and acquisition reuse. The +release guide owns the implemented operational procedure. `tooling/performance-report.toml` binds release eligibility to the full measurement declaration and owns the current report and immutable history. @@ -263,7 +272,13 @@ All views require explicit input ownership. the measurement, report and README policies. Installed `performance prepare`, `release` and `readme` commands own those separately retriable phases. The former measurement/publication implementations and root `scripts/` directory are retired. -The installed CLI owns orchestration, validation and publication transactions. +Shared public APIs own acquisition, validation and publication transactions. +`tooling/release_candidate.py` owns thin repository orchestration: immutable +owner records outside the checkout, pre-promotion capture, named final acceptance, +original archive derivation, exact draft staging and authenticated transfer. +It does not implement a comparison engine, archive parser or CI verifier. +`tests/tooling/test_candidate_transfer.py` exercises actual privileged workflow +blocks and exact selection/credential boundaries. Working-tree measurements now include nonignored untracked regular files. `tests/tooling/test_performance_workflow.py` exercises actual recipes and scientific policy. Shared failure matrices remain upstream. The @@ -285,9 +300,6 @@ validates separately. `tests/tooling/test_semgrep_integration.py` exercises the locked fixture command and its failure statuses. Git-ignored dependencies, environments, caches, and build outputs remain outside production scans. -`.github/actions/prepare-release-benchmarks/action.yml` groups tool installation, -input validation, and inventory under the release workflow's shared setup timeout. - `.github/workflows/ci.yml` owns native checks, default/exact Cargo dry-runs and signed main-push validation evidence. `tooling/release-evidence.toml` supplies native/Cargo and signer policy to the installed verifier; @@ -295,8 +307,12 @@ native/Cargo and signer policy to the installed verifier; `tooling/release-assets.toml` declares the draft-target identity for the installed `release-assets` recipe. Privileged benchmark jobs continue to use installed asset commands from `RUNNER_TEMP`, without executing checkout code. -`.github/workflows/publish-crates.yml` owns the reviewed-release event, reuse of -that exact CI attempt and artifact, deployment approval and OIDC upload. The publishing declaration +`.github/workflows/release-benchmarks.yml` separates no-checkout draft retrieval, +read-only candidate verification and no-checkout attachment. It does not measure. +`.github/workflows/publish-crates.yml` consumes a second maintainer dispatch with +independent canonical-asset and full acceptance assertions. It owns approval, +the immutable approval receipt, explicit native-evidence renewal, revalidation +and OIDC upload. The publishing declaration in `pyproject.toml` selects `la-stack`, native checks and the Criterion asset; shared release gates own generic identity, ancestry and registry checks. `tests/tooling/test_release_publishing.py` owns installed-package consumer checks diff --git a/docs/dev/benchmark-candidate-review.md b/docs/dev/benchmark-candidate-review.md new file mode 100644 index 0000000..d576f1f --- /dev/null +++ b/docs/dev/benchmark-candidate-review.md @@ -0,0 +1,235 @@ +# Benchmark candidate and resume review + +Independent Astra architecture checkpoint on 2026-11-07 for +[#325](https://github.com/acgetchell/la-stack/issues/325) and +[#323](https://github.com/acgetchell/la-stack/issues/323), requested by the +maintainer before workflow changes. Reviewed the working tree at HEAD +`f520e2f59574774454a739c7dfa810f3d1845420` and installed research-repo-tools +`13af6b60c9bc1712b329bfe9504c7a54840e043c`. This records a conditional design +decision and bounded experiments, not completed adoption or release readiness. + +The follow-up below records adoption at the new pin. Earlier findings remain +as dated evidence and are not current blockers. + +## Contents + +- [Current adoption](#current-adoption) +- [Decision](#decision) +- [Resume contract](#resume-contract) +- [README input ownership](#readme-input-ownership) +- [Release ordering and trust](#release-ordering-and-trust) +- [Consumer acceptance](#consumer-acceptance) + +## Current adoption + +On 2026-11-07, upstream commit +`055353c630604b1ae3438c1eaf534ba85e067a7e` +([PR #205](https://github.com/acgetchell/research-repo-tools/pull/205)) supplies +the accepted #203 and #204 contracts. The preserved installed-package probes +pass unchanged. Schema 6 now retains completed suite pairs within an interrupted +trial; phase-specific sources truthfully declare baseline README input while +allowing current README output. + +The consumer uses one sequential trial with all 1,403 library cases and 16 +paired LU peers per phase. Current native gates inspect debug/release dependency +files and reject README compiler input before timing. Historical current tags +that still compile README are intentionally ineligible for this full plan; +non-exact diagnostics retain conservative input declarations. + +The local adapter captures original raw bytes, preparation, candidate assertions +and publisher receipt before promotion in a separate owner-controlled directory. +Final acceptance uses an isolated installed supplier and a clean final commit. +Named immutable review selections permit explicit review of a newer native +attempt without overwriting the original acceptance. Capture recovery from +scratch is explicit; normal retries never adopt untrusted incoming assertions. + +Astra accepted two maintainer dispatches as the authenticated handoff. The first +uses independent original asset IDs/hashes to retrieve, verify and attach inert +evidence; the second additionally pins the canonical attached archive and full +acceptance across deployment approval. Write-scope jobs execute no checkout code. +The native signer remains main-push-only. No new upstream capability was needed +for this consumer orchestration. + +Follow-up review found and corrected local immutable-record bootstrap, runtime +isolation, capture recovery, final review selection, draft retry identity, +canonical asset verification, title/notes retention and Just variadic forwarding. +Review also identified a recovery gap after immutable GitHub publication: +native-evidence renewal could not replace the original acceptance asset. +The Cargo dispatch now takes an independently reviewed full acceptance digest, +freezes that exact receipt before approval and revalidates its bytes afterward. +The original archive, raw observations and all release assets stay unchanged. +Astra's final scoped review accepted this renewal mechanism without another +supplier change. Actual shell regression tests also require explicit failure +exits for the artifact guards, including on macOS's older Bash. +The full synthetic candidate path passes through capture, publication, final +binding, retry, archive derivation, hosted receipt verification and explicit +native-evidence renewal before and after approval. It uses the installed shared +candidate/archive verifiers and checks unchanged original bytes throughout; +provider responses and timings are synthetic. Both scratch and retained README +publication regressions pass. Final macOS `just ci` passes with 367 Python tests, +1,143 Rust tests, default/exact doctests, benchmark compilation and correctness +checks, examples and the required static validators. Windows-targeted Python +typing and explicit rule scans of the two new Python files also pass. +No additional supplier defect was found. Native consumer Linux/Windows CI, +a published supplier pin, real timing evidence and actual hosted release +handoffs remain separate release requirements. No dispatch or upload has run. +The release guide owns the operational commands and remaining publication gates. + +For the v0.5.2-to-v0.5.3 availability case, acquisition falls from 2,838 local +observations plus 1,403 hosted repeats to the same 2,838 local observations. +At eight nominal warmup/measurement seconds per case, this removes 187.1 +aggregate minutes (565.5 to 378.4). The old hosted feature suites ran in parallel, +so their nominal critical path was the slower 98.1-minute default suite plus +setup and assembly, not their summed sampling time. New verification/transfer +work remains. These are planning calculations; no comparable real campaign +establishes elapsed-time savings. + +## Decision + +Retain local maintainer-owned Step 1 acquisition. Preserve all 1,403 library +cases, availability, paired LU peers, sampling settings and original raw bytes. +Use one explicit sequential trial for an initial schema-6 migration; this +preserves the existing measurement budget and makes no counterbalancing claim. +README views must explicitly select trial 1. More trials require a scientific +reason and resource decision; they do not solve interruptions within a trial. + +The shared candidate API supports the intended local acquisition, publication +receipt and final-commit binding. Accept that architecture subject to the input +and trusted-handoff conditions below. Joint resume/reuse completion requires +the supplier decisions in +[research-repo-tools#203](https://github.com/acgetchell/research-repo-tools/issues/203) +and [#204](https://github.com/acgetchell/research-repo-tools/issues/204), both +assigned to v0.2.7. Keep the existing release procedure until its replacement +passes acceptance. A dependency update or safe fallback does not close #323. + +## Resume contract + +The upstream [paired-suite decision](https://github.com/acgetchell/research-repo-tools/pull/175) +accepts a whole suite with both available phases as a durable schema-5 unit. +An incomplete pair is recollected in full after retained-unit validation; +completed raw bytes and original host/session attribution remain unchanged. + +Candidate reuse requires schema 6 and complete-run v5. At the reviewed pin, +its journal retains only complete trials. A started trial without its complete +receipt rejects resume because it may contain unretained timings. The scheduled +engine does not pass a nested checkpoint to suite acquisition. With one trial, +no incomplete campaign can recover its already completed suites. + +A bounded installed-package pytest probe uses two synthetic paired suites, +`shared` and `later`. It interrupts before `later/current`, after verifying +the command log contains `shared/baseline`, `shared/current` and +`later/baseline` measurements. The same success contract runs against both +schemas: schema 5 passes without repeating `shared`; schema 6 fails with +`interrupted trial may contain unretained timings`. The failing success +assertion is preserved, not changed to accept a fresh acquisition. + +The probe and unchanged supplier fixture are retained under +`target/research-tools-rehearsal/13af6b60c9bc1712b329bfe9504c7a54840e043c/consumer/`. +They are ignored diagnostic artifacts, not ordinary test-suite inputs or +performance evidence. #203 contains the reproduction and required contract. + +The supplier must independently review recovery granularity, original session +and timestamp retention, interruption history, phase order and incomplete-pair +recollection. Retain rejection of ambiguous or corrupt work; deleting the +existing guard would not establish safe recovery. Final resume wiring waits +for this accepted public contract. + +## README input ownership + +Both current source and v0.5.2 use an unconditional +`#![doc = include_str!("../README.md")]`. README is therefore a compiler input. +The candidate contract requires truthful input declarations and prohibits +input/output overlap, including documents read by builds. Its final comparison +checks inventories from every observation, including baseline phases. + +Existing Rust dependency files and a fresh Rust 1.99.1 compiler probe confirm +the unconditional include reads README during ordinary compilation. A tiny +`cfg_attr(doc, doc = include_str!(...))` control excludes it from ordinary +compiler dependencies. This control does not establish native benchmark, +rustdoc or package equivalence for la-stack. + +A separate installed-API probe loads the full la-stack plan, adds README to +the source inputs, and selects a sequential schema-6 schedule. Declaring README +as an output then correctly rejects with `declared output overlaps measured +inputs`. Omitting the input to bypass this check would violate the contract. + +A current-source `cfg(doc)` change is an option to investigate, not an accepted +complete fix: the immutable baseline still includes README, and source patterns +are shared across phases. #204 requests a supported phase/source-aware contract +or demonstrated consumer architecture. Do not overlay changed library source +as harness, weaken input ownership, or remove crate documentation. Any accepted +consumer change needs both-phase compiler-input proof, default/exact rustdoc +and example validation, native gates and final package evidence. + +## Release ordering and trust + +The following is the reviewed target design; it is not yet executable guidance. +The [release guide](../RELEASING.md) remains the operational owner. + +1. Finish versions, dependency pins, lockfiles, source, harness, commands and + output declarations before Step 1. Any subsequent non-output tree change + invalidates reuse, including a new supplier pin. +2. Acquire once through the public release API. Its `before_promotion` hook + freezes `CandidatePolicy(commit=None, report=...)`, the independently retained + acquisition digest, publication receipt and separate receipt digest. The + hook does not publish. A failed hook leaves raw evidence for a direct retry. +3. Keep the owner record outside candidate-controlled files with an immutable + backup or authenticated transport. An ignored `target/` directory alone is + not an authentication boundary. A checksum stored beside the candidate does + not authenticate that candidate. Preserve local ownership throughout. +4. Render report/history through the exact shared-publisher receipt and README + from the selected retained trial. Retry rendering or promotion directly; + do not call acquisition again. Do not exempt arbitrary JSON or directories. +5. After merge and final native CI, run candidate verification before tagging. + Independently select the final clean commit and supply the original trusted + digests. Retain acceptance outside committed source. Step 2 must not measure, + discover, preflight, rebuild, tag or publish; it may write its requested + verification receipt. +6. Derive and verify the archive through public APIs. Replace the hosted second + acquisition only after exact artifact selection and attachment pass tests. + Keep original measured revisions, host, sessions and timestamps distinct + from final-release acceptance. +7. Freeze the full acceptance and exact native CI/artifact selection before + approval; revalidate that same selection after approval and before release + credentials. Preserve protected environments, draft/tag checks, package + guarantees, no-overwrite checks and credential separation. + +The existing release publication event transfers native CI identity, not +independently authenticated candidate assertions. The recommended transport is +a maintainer-authorized workflow dispatch supplying acquisition, publication, +acceptance and archive digests from the separate owner record. Authenticate the +allowed workflow, dispatcher, event, ref, commit and exact run/attempt/artifact; +compare separately transported files against those assertions. Later jobs must +retrieve that frozen selection, never a latest-by-name artifact or digests +computed from the candidate being checked. + +Actual handoff ownership and authentication require consumer implementation and +demonstration. No new service, signer or protected-environment policy is assumed +to exist. Keep generic verification upstream and use thin consumer orchestration. +Local capture/receipt prototypes can proceed independently, but they cannot +substitute for authenticated hosted publication. + +## Consumer acceptance + +- Preserve independent inventories, availability and same-suite peer attribution + in bounded fixtures. Synthetic samples establish workflow behavior only. +- Exercise one Step 1 acquisition through publication, final clean merge, + signed native evidence, candidate acceptance, archive and attachment retries. + Compare retained raw bytes and original provenance throughout. +- Put failure sentinels on acquisition, discovery and preflight in Step 2, + report/README retries, archive derivation and approval-time revalidation. +- Reject input, declaration, dependency, source, harness, compiler-relevant + document and frozen generated-output drift before publication or new timing. +- Reject replaced owner assertions, receipts, acceptance files, run attempts + and immutable artifacts, including replacement after initial approval checks. +- Prove eligible missing-suite recovery under the accepted supplier contract; + retain partial-output, corruption, ordering and writer-ownership rejection. +- Complete focused consumer checks and native platform evidence before replacing + the release workflow. Development pins remain forbidden at actual release + boundaries until a published supplier is adopted and revalidated. + +The original architecture checkpoint performed no real timing campaign, release +upload, publication, repository Git mutation or production resume/reuse wiring. +The current adoption section records the later consumer implementation. Group 2's +separate [validation record](benchmark-workload-review.md#validation-evidence) +retains its own source and supplier identities. diff --git a/docs/dev/benchmark-workload-review.md b/docs/dev/benchmark-workload-review.md new file mode 100644 index 0000000..093b4cd --- /dev/null +++ b/docs/dev/benchmark-workload-review.md @@ -0,0 +1,382 @@ +# Release benchmark workload review + +Consumer review for [#322](https://github.com/acgetchell/la-stack/issues/322), +with availability and progress integration findings for +[#320](https://github.com/acgetchell/la-stack/issues/320) and +[#324](https://github.com/acgetchell/la-stack/issues/324). +Initial review on 2026-11-05 at source +`f520e2f59574774454a739c7dfa810f3d1845420` used the +`public-release-and-lu-peers-v3` policy. The 2026-11-06 integration uses +`public-release-and-paired-lu-peers-v4` without removing library workloads. + +## Contents + +- [Selection decision](#selection-decision) +- [Required coverage](#required-coverage) +- [High-cost combinations](#high-cost-combinations) +- [Optional and excluded measurements](#optional-and-excluded-measurements) +- [Availability adoption boundary](#availability-adoption-boundary) +- [Budget and progress](#budget-and-progress) +- [Validation evidence](#validation-evidence) +- [Remaining acceptance](#remaining-acceptance) + +## Selection decision + +Retain all 1,403 current library cases: 736 default and 667 exact. No workload +removal or sampling reduction is proposed. The exact required IDs remain in +[`performance-policy.toml`](../../tooling/performance-policy.toml); this review +explains their coverage rather than introducing another selection mechanism. +The full native catalog remains available for explicit diagnostics. + +The two largest repeated families measure different caller costs and numerical +regimes. Their counts alone do not establish equivalence. No comparable +end-to-end timing campaign was performed for this review, and no speedup is +claimed. Independent Astra review on 2026-11-05 accepted this no-removal +conclusion, subject to the required native discovery and correctness checks. +That review accepts scientific coverage, not minimality or measured runtime. + +## Required coverage + +Each row below is required in the current release selection. Construction and +conversion stay timed when they are part of the named public operation. Fixture +generation and independent oracle evaluation stay outside timing. The native +targets and filters are owned by [`performance.just`](../../tooling/performance.just). + +| Native family / feature | Cases | Calling forms, algorithm regimes and dimensions | Independent correctness evidence | +| --- | ---: | --- | --- | +| `angle` / default | 168 | Prepared vectors, checked borrowed slices, construction plus angle; 14 dense/sparse, endpoint, mixed-scale and subnormal families at D=3–6 | Analytical references and integer Gram/exterior-product oracles in `benches/common/angle.rs`; `tests/angle_bench_inputs.rs` and `tests/vector_angles.rs` | +| `column_basis` / default | 55 | Square/tall/wide, deficient, near-dependent, rescaled and affine constraints; shapes through 8 × 14 and 6 × 7 | Expected rank and pivot sets in the native harness; independent exact minor checks in `tests/column_basis.rs` | +| `exact` binary64 inputs / exact | 81 | Determinant, sign, solve, strict/rounded results; closed forms through D=4 and elimination at D=5; fixed, deterministic corpus, near-singular, large-entry and Hilbert inputs | Validated exact fixtures and independent rational Gaussian elimination; `tests/exact_bench_config.rs` | +| `exact` canonical conversion / exact | 128 | Four scalar/vector result forms, eight representability/size families at D=2–5; dyadic, non-dyadic, subnormal, underflow, overflow boundary and wide integers | Known exact conversion values and typed error expectations in the native fixture gates and exact tests | +| `exact` rational inputs / exact | 63 | Determinant, sign and solve; small, 256-bit and 1,024-bit inputs at D=2–8, preserving closed-form/elimination and integer-size regimes | Independent rational Gaussian determinant/solve and manufactured solutions | +| `gram` / default | 56 | Public Gram construction for square and embedded vectors through coordinate dimension 8; orthogonal, dependent, near-dependent and mixed-scale | Independent integer matrix-product oracle in `benches/gram.rs` | +| `interval` / default | 43 | Scalar arithmetic, lifted assembly and prepared determinant/sign; exact, cancellation, zero, underflow, sparse lifted and dense conclusive/inconclusive regimes at D=3–7 | Fixture-specific expected enclosures/signs in the native harness; interval unit/property tests | +| `interval_solve` / default | 35 | Five division/reciprocal cases and five 32-system solve corpora at D=1–6; dense, uncertain, near-singular, singular and range-exhaustion outcomes | Manufactured solutions plus independent rational Gauss-Jordan checks in `tests/interval_solve.rs` | +| `linear_form` / default | 140 | Dot/affine-difference certificates and endpoints at D=2–6; dense, sparse, cancellation, underflow/overflow; construction, prepared and construction-inclusive projection batches | Independent fixture expectations and certified reduction unit/property tests | +| `polynomial` / exact | 5 | Full sparse determinant polynomial plus borrowed leading term at orders 4–8; allocation/destruction included | Sparse permutation enumeration of every coefficient in `tests/polynomial_bench_inputs.rs` | +| `public_api` / default | 90 | Construction, access, reset-copy mutation, symmetry, tolerances, direct determinant/error-bound and interval queries at D=2–5; dimensionless scalar operations once | Known values and typed failure contracts in library/integration tests, with harness assertions | +| `public_api` / exact | 42 | Rational storage/construction/mutation and scalar strict/rounded conversion; D=2–5 where dimensional | Exact-value and conversion/error tests; native fixture assertions | +| `rational_dot` / exact | 54 | Prepared vector and checked slice; non-dyadic, cancellation and 512-bit integer families at lengths 2–8, 14 and 22 | Analytical reciprocal-product sums; `tests/rational_dot.rs` | +| `rational_solve` / exact | 294 | Construction, prepared solve and runtime-dispatch adapter; seven matrix structures, two coefficient families, D=2–8 | Manufactured exact solutions, independent Gaussian control and residuals; `tests/exact_bench_config.rs` and `tests/rational_structured_solve.rs` | +| `vs_linalg` library rows / default | 149 | LU/LDLT; factorization plus one solve/determinant and prepared operations; dot/norm regimes at D=2–64; five additional D=8 pivoting, conditioning and range cases | `benches/comparison/tests/vs_linalg_inputs.rs`, factorization unit/property tests | + +The original README policy additionally required 16 baseline-origin peer cases: +nalgebra and faer factorization plus one solve at D=2, 3, 4, 5, 8, 16, 32 and 64. The matching +eight la-stack cases are already included above. Same-environment attribution +and the timed boundary must survive an orchestration change: LU factorization +plus one solve from prepared inputs, including required workspace and result +destruction. Matrix/vector input representations are constructed outside timing. +Hosted library-only evidence cannot replace the local denominator. The +[availability decision below](#availability-adoption-boundary) replaces the +baseline-only peer selection in new acquisitions with a paired suite. + +Default constructors share their explicit construction workload. Formatting, +error factories, enum queries, indexing/dispatch macros and re-exported traits +are tested directly or exercised by their containing public workflow; they do +not require separate timing rows. This is a public-workflow coverage argument, +not a claim that every possible parameter combination has been measured. + +## High-cost combinations + +`rational_solve` has 7 structures × 2 coefficient families × 7 dimensions × 3 +calling forms = 294 cases per revision. Diagonal, upper/lower triangular and +row-permuted triangular inputs exercise structure recognition and substitution; +sparse and dense inputs retain elimination behavior. Dyadic versus non-dyadic +coefficients change denominator work. D=2–8 retain separate const-generic +instantiations and the library's dimension/dispatch boundaries. The adapter +includes construction, dimension dispatch and output collection; the prepared +solve does not. Construction alone isolates a caller-visible cost. No equivalent +representative has been established that justifies removing a combination. + +`angle` has 14 fixtures × 4 dimensions × 3 calling forms = 168 cases per revision. +Identical/antipodal endpoints, nearly parallel/antipodal inputs, sparse support +patterns, independent scaling and subnormal behavior are different numerical +questions. Prepared finite vectors avoid checked-slice validation; construction +plus angle includes two constructors. A single generic kernel does not prove +that distinct const-generic compiled dimensions or caller forms have equal cost. + +Together these families contribute 924 paired cases, or 123.2 nominal minutes +at 8 seconds per case. That identifies review cost; it is not evidence of +redundancy or observed runtime. Correctness/property tests remain comprehensive +regardless of any future timing-selection decision. + +## Optional and excluded measurements + +| Classification | Cases / selectors | Reason | +| --- | --- | --- | +| Optional diagnostics, excluded from mandatory release timing | `angle_d*/stable_control`; hand-written Gram controls | Reference kernels omit or rearrange public validation/work and answer implementation questions | +| Optional diagnostics, excluded from mandatory release timing | `rational_solve/*/legacy`; rational Gaussian controls and exponent-spread studies in `exact` | Independent reference/implementation investigation; correctness oracles remain required outside timing | +| Optional diagnostics, excluded from mandatory release timing | Broad nalgebra/faer operations outside the 16 LU-solve peers | Existing README question is factorization plus one solve through D=64; full comparison remains explicitly runnable | +| Excluded only on historically unavailable revisions | 55 column-basis, 35 interval division/solve, 54 rational-dot and 5 polynomial cases before v0.5.2 | Missing public APIs; all 149 stay required on supported revisions, never optional because compilation failed | +| New exclusions proposed by this review | None | No scientifically justified reduction established | + +## Availability adoption boundary + +The initial review used **research-repo-tools 0.2.6**, resolved from PyPI. +The development branch now tests exact upstream commit +`055353c630604b1ae3438c1eaf534ba85e067a7e`; its unchanged 0.2.6 version label is +not a published-artifact identity. Follow the +[development pin policy](../../tooling/README.md#shared-capability-adoption) +before release. On 2026-11-05 the +[exact-version registry metadata](https://pypi.org/pypi/research-repo-tools/0.2.6/json) +reported non-yanked wheel and source artifacts whose SHA-256 hashes match +the original `uv.lock`. The package-identity integration test now checks the +exact manifest, lock and installed commit, including isolated release runtimes. + +Its [published schema-5 contract](https://github.com/acgetchell/research-repo-tools/blob/v0.2.6/docs/DECLARING_PERFORMANCE.md) +provides inclusive `since` bounds resolved from captured release/source receipts +before overlay or native commands. This is the intended replacement for the +permanent `[baseline].current-only` exclusions. The required resolved inventory +is 1,254 library cases on v0.5.0/v0.5.1 and 1,403 on v0.5.2 and later, plus the +16 peers on each phase. Reading an overlaid `Cargo.toml` cannot establish the baseline +release. Supported compilation/discovery failures must still fail. + +**The original phase selection was incompatible with schema 5.** A bounded synthetic +probe of the installed public `parse_suite_plan`, `measure_suites`, +`validate_run_policy` and `comparison_incompatibilities` interfaces used one +accepted paired suite, identical commands, full raw samples and known local +host observations. Library-against-library cross-release comparison and +same-phase peer comparison were eligible. Comparing baseline +`d2/nalgebra_lu_solve` with current `d2/la_stack_lu_solve` returned: + +```text +workload IDs differ within the semantic contract +``` + +This fails even before accounting for the different commands needed to acquire +peers only once. The interface also requires matching cross-phase commands and +rejects cross-unit ratios. Splitting peers into a separate unit, relabeling their +phase, reusing one ID for different implementations, or suppressing ratio checks +does not preserve the consumer contract. Measuring peers on both revisions +would change acquisition ownership and budget, and still would not make the +baseline-peer/current-library comparison eligible. + +Independent Astra review recommended preserving this deliberate supplier +restriction. The new declaration puts the eight la-stack LU-solve cases and +16 peer cases in one paired schema-5 suite with identical native commands on +both revisions. Those eight library cases are excluded from the other suites to +preserve single ownership. The README selection compares current-phase peers +with current-phase la-stack; the suite also retains baseline/current library ratios. +Keep historical baseline-origin evidence unchanged and attribute each new +observation to its actual phase. + +This adds 16 peer measurements, or 128 nominal seconds, about 0.6% of the full +planned campaign. The repeated peers serve distinct paired evidence. Closer +collection reduces exposure to phase/order drift, but ratios remain descriptive. +No widening of supplier ratio eligibility is needed. A future insistence on +baseline-only peers with cross-phase ratios would instead require a separately +reviewed contract for operation/input equivalence, implementation identities and +allowed selection differences, with all provenance safeguards preserved. + +The full declaration now uses seven schema-5 suites: shared default, shared +exact, paired LU solve, and four suites introduced at v0.5.2. Native discovery, +correctness gates, independent inventory tests, series selections and progress +budgets have been reconciled. At commit `a4640b0`, end-to-end acceptance was blocked by +[research-repo-tools#196](https://github.com/acgetchell/research-repo-tools/issues/196): +the publisher read dependency-version labels from aggregate phase provenance, +but schema 5 records them in the owning suite observation. Both scratch and +retained plot publication failed despite valid captured dependency versions. +The publication regressions and version labels were preserved. + +Preparation had a separate early-validation gap tracked in +[research-repo-tools#197](https://github.com/acgetchell/research-repo-tools/issues/197). +The schema-5 selector skipped active suites' minimum-release checks and could save +a successful preparation for v0.4.6 despite the v0.5.0 floor. A correctly +versioned temporary Git fixture and an independent public-API reproducer both +confirmed the gap. Acquisition still rejected that pair before native commands; +the finding did not establish invalid measurement acceptance. + +Commit `a8dc578a7583f1e4d811a5bad1ceb47ae28949e4` +([PR #199](https://github.com/acgetchell/research-repo-tools/pull/199)) supplies +both fixes. The unchanged public-API reproducers pass against its installed +package. The current development pin also supplies the root dependency-source +policy from PR #198; that declaration is retained with the harness inputs. + +This report does not claim #320 complete or authorize a long measurement +campaign. Upstream contract history is +[research-repo-tools#175](https://github.com/acgetchell/research-repo-tools/pull/175). +PR [#190](https://github.com/acgetchell/research-repo-tools/pull/190), inspected at +`19476650960c4a441121a0a8b98ee2cc0e75ebe4`, addresses committed publisher evidence +and release summaries. Its `suite_runs.py` is byte-identical to installed 0.2.6, +so it does not change this intentional comparison policy. + +## Budget and progress + +| Inventory | Measurements | Nominal warmup plus sampling | +| --- | ---: | ---: | +| Existing v0.5.1 baseline policy: 1,254 baseline + 16 peers + 1,403 current | 2,673 | 356.4 min | +| Reviewed paired-peer plan with v0.5.0/v0.5.1 baseline: 1,254 baseline + 1,403 current + 32 peers | 2,689 | 358.5 min | +| Reviewed paired-peer plan with v0.5.2-or-later baseline: 1,403 baseline + 1,403 current + 32 peers | 2,838 | 378.4 min | +| Hosted current library only | 1,403 | 187.1 min | + +These totals use the existing 3-second warmup and 5-second measurement target; +they exclude preparation, compilation, gates, analysis and publication and are +not deadlines. No new observed acquisition cost is available. + +The canonical release recipe exposes the installed reporter's human/JSON/none +mode and heartbeat cadence. Native diagnostics remain on their original streams; +the reporter reads no native logs and counts only validated phase results. +The full schema-5 plan reports resolved per-suite inventory, nominal budget, +completed/remaining cases, stage, elapsed seconds and unknown ETA for unseen +work. The former hardcoded native budget line has been removed. Synthetic +consumer checks verify 2,689 validated observations for a v0.5.1 baseline and +the suite/phase counts at the v0.5.2 availability boundary. +The shell runtime record remains the observed recipe-body duration and exit code. + +The initial consumer gate-failure regression exposed a supplier defect: schema-2 +acquisition raises `ValueError("baseline preflight gate failed")` correctly, +but its reporter closes the gate stage with `finished` before checking the +nonzero exit status. It emits no `failed` event through `measure_prepared_pair`. +The regression requires the failure event and zero accepted cases. +The defect is tracked in +[research-repo-tools#191](https://github.com/acgetchell/research-repo-tools/issues/191). +The inspected #190 head has exactly the same `common_measurement.py` blob as +installed 0.2.6 (`27f1692981cb590ab63482ae538239b324f1a9d2`), so that PR does not +fix this failure. Commit `ef783cac608bc0d2d4657052c20e663f2d45b232` fixes it; +the preserved prepared-pair and baseline/current schema-5 regressions pass. +Keep those regressions and adopt the supplying published release before final +publication; do not add consumer event rewriting or relax gate acceptance. + +See [the benchmark guide](../BENCHMARKING.md#release-acquisition-policy) for +commands and retry boundaries. Report/README retries consume retained evidence +without measurement. Heartbeats, nominal budgets and estimates never replace +raw-file, sampling, correctness or freshness validation. + +## Validation evidence + +The initial local macOS/Python 3.15 checks on this draft were: + +- `just check`: passed, including documentation, spelling, configuration, + Python types/lint, shell, Rust checks and repository static analysis. +- Both affected consumer modules: 45 passed, 1 failed in 279.73 seconds. + The failure is the preserved gate-status regression described above; + report/README retry and source/declaration freshness checks passed. +- Exact manifest/lock/installed-package identity test: passed. +- Windows-targeted strict type checking of both changed Python modules: passed. + This is static evidence, not native Windows execution. +- `uv lock --check` and `git diff --check`: passed. + +The 2026-11-06 rehearsal against installed commit `a4640b0` additionally verifies: + +- 249 supplier public-consumer tests and 734 subtests passed outside the supplier + checkout, covering acquisition, resume, rendering, release handoffs and the + development-pin policy with synthetic samples. +- Eight focused consumer availability checks passed: v0.5.0, v0.5.1, exact + v0.5.2 and later boundaries, captured source receipts, unsupported releases, + and rejection of incompatible historical evidence without rewriting it. +- Native discovery and correctness gates passed on exported real v0.5.1 and + current sources with separate compiler storage. Required inventories were + exactly 1,270 baseline and 1,419 current cases, including 16 peers per phase; + no benchmark timing samples were collected. +- Ordinary toolchain verification accepts the exact development commit. + Release-only dependency validation rejects it before acquisition or fetching, + as required by upstream #193. CI omits release signing for development pins. +- The public publication regression for upstream #196 fails in isolation with + a captured dependency version and two synthetic samples. Downstream scratch + and retained publication checks reproduce the same failure. +- The public preparation regression for upstream #197 fails without any native + command or provider access. The full CLI separately saves an unsupported pair + in a temporary Git fixture. Both upstream issues target v0.2.7. +- `just check` passed, including Markdown/spelling, guide documentation builds, + Python lint/types, configuration/workflow checks and static analysis. + Windows-targeted type checking passed; this is not native Windows execution. +- The full downstream Python run had 316 passes and seven failures. Focused + reruns after local assertion/configuration fixes passed 69 preparation/release + checks and both source/harness freshness checks. Four regression failures + remained at that pin: three publication paths blocked by #196 and unsupported + preparation accepted by #197. That downstream suite was not green. + +The 2026-11-06 rehearsal against installed commit `a8dc578` verifies: + +- Both unchanged public-API reproducers for #196 and #197 pass. +- 256 supplier public-consumer tests and 756 subtests pass outside the supplier + checkout, covering synthetic acquisition, resume, rendering, release handoffs, + dependency-source policy and the preserved regressions. +- All 125 focused downstream workflow, preparation, toolchain and publication + checks pass. Scratch and retained publication preserve the captured dependency + versions; unsupported preparation fails before native work. +- A separate consumer regression at the exact v0.5.2 availability boundary + verifies that failed column-basis discovery remains a failure with zero + accepted cases. It cannot silently exclude a supported workload or proceed + to gates and acquisition. +- The root dependency-source declaration covers Python tooling and the Cargo + workspace, including the private comparison member. Exact-pin checks accept + the development identity; release-entry checks reject it without creating + evidence or fetching source revisions. +- All 89 native Rust, manifest, lock, toolchain and driver inputs match the + preserved native-discovery receipt above. The changed shared-tooling pin and + dependency policy have new evidence identities; the old receipt is not + relabeled as a new measurement. +- `just check`, Python lint/types, Windows-targeted strict type checking, + `uv lock --check` and patch whitespace checks pass. Windows type checking is + static evidence, not native Windows execution. +- Final `just ci` passes: 325 Python tests and 1,143 Rust tests, plus default/exact + doctests, benchmark compilation, exact-benchmark test mode, examples and the + repository's static checks. No timing campaign is part of this validation. + +A subsequent maintainer-requested `just update` completed tool setup and Cargo +maintenance but exposed a separate shared-updater gap: +[research-repo-tools#200](https://github.com/acgetchell/research-repo-tools/issues/200). +Python pin discovery rejected the exact Git source in the preserved tooling +group before resolution. An installed public-API reproducer confirmed that +failure without modifying its manifest or creating a lock. Explicit uv lock +refresh and synchronization succeeded with unchanged pins and lock contents; +the normal update command at `a8dc578` required the upstream fix. This maintenance +path was not part of the preceding acquisition/publication acceptance suite. + +On 2026-11-07, commit `13af6b60c9bc1712b329bfe9504c7a54840e043c` supplies +[the #200 fix](https://github.com/acgetchell/research-repo-tools/pull/201). +The unchanged reproducer passes, and the complete normal `just update` succeeds +while preserving the exact supplier commit and inherited tool pins. Its full +lock refresh advances only `sse-starlette` from 3.6.0 to 3.7.0. The manifest, +lock, isolated release runtimes and consumer identity assertions use the new +commit; prior rehearsal receipts retain their original identities. +The 72 focused downstream toolchain/publication tests and final `just check` +also pass on this pin. An initial online workflow audit hit a GitHub timeout; +the required online retry and subsequent complete check both passed. This +does not relabel the preceding full `just ci` receipt as a run on the new pin. + +The 2026-11-07 continuation uses +`055353c630604b1ae3438c1eaf534ba85e067a7e` from upstream PR #205. +`just update`, locked synchronization and the three preserved #203/#204 +regressions pass. The full declaration now uses schema 6 and one sequential +trial; an interrupted trial retains completed suite pairs. The consumer resume +fixture preserves completed LU observations, reacquires only the interrupted +column pair and rejects changed input before invoking commands. + +Native discovery and correctness gates pass for all seven suites at the +captured v0.5.2 baseline and working-tree current source: 1,419 cases per phase, +including all 1,403 library cases and 16 LU peers. Dependency files confirm +README as a baseline compiler input and its absence from both current build +profiles. These receipts precede final hosted-renewal orchestration edits; +those edits do not alter the native workload, sampling or gate commands. +The independent current-input guard has six passing failure/success fixtures. + +The focused hosted-transfer and publication suite has 90 passing tests, and +all nine command-discovery tests pass. Host and Windows-targeted Python type +checks, `just check`, Markdown validation and rustdoc checks pass. The final +comprehensive validation and synthetic renewal receipt are recorded in the +[candidate review](benchmark-candidate-review.md#current-adoption). + +These results establish discovery and orchestration behavior, not performance. +No real timing campaign, native Windows/Linux execution, release publication or +repository Git mutation was performed. Temporary synthetic Git repositories +are test fixtures only. + +## Remaining acceptance + +Group 2's development integration now passes local bounded acceptance. The +remaining release requirements are: + +- Before a real release, replace development pins with the normally published + supplier version, resolve/sync the lock, and revalidate the changed inputs. + Existing rehearsal evidence cannot be relabeled for the published pin. +- Complete native platform checks and any separately authorized real campaign. + Preserve historical report/raw bytes under their original plan identities. + +The bounded rehearsal does not establish release readiness. No issue is closed +by this report. + +The subsequent [candidate/resume architecture checkpoint](benchmark-candidate-review.md) +records the separate prerequisites for #325 and #323. Those findings do not +change group 2's scientific selection or establish completion of later groups. diff --git a/docs/dev/docs.md b/docs/dev/docs.md index 8c5b49f..b7e0e5e 100644 --- a/docs/dev/docs.md +++ b/docs/dev/docs.md @@ -90,7 +90,7 @@ paths; never rename a generated report without updating its source workflow. ## README and rustdoc guides `src/lib.rs` includes the README with -`#![doc = include_str!("../README.md")]`, so README examples are also the +`#![cfg_attr(doc, doc = include_str!("../README.md"))]`, so README examples are also the docs.rs landing-page examples. Keep the README quickstart and brief capability descriptions discoverable. Put diff --git a/justfile b/justfile index 8449c28..eca238e 100644 --- a/justfile +++ b/justfile @@ -8,6 +8,8 @@ set shell := ["bash", "-euo", "pipefail", "-c"] _run := "uv run --locked --no-sync --no-python-downloads research-repo-tools toolchain run --" +_pytest_parallel := "-n " + quote(env("LA_STACK_TEST_WORKERS", "4")) + " --dist worksteal --max-worker-restart 0" + # Semgrep misreads Python 3.15's positive RLIM_INFINITY on Linux. # Keep its inherited stack finite until semgrep/semgrep#11979 is fixed. semgrep_stack_limit := if os() == "linux" { 'if [[ "$(ulimit -Hs)" == unlimited ]]; then ulimit -s 1048576; fi;' } else { "" } @@ -52,6 +54,15 @@ _ensure-uv: _ensure-yamllint: _ensure-uv uv run --locked yamllint --version >/dev/null +# Isolate the supplier runtime from the observed candidate checkout. +[positional-arguments] +[private] +[script("uvx", "--python", "3.15", "--from", "research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e", "python", "-I")] +_release-candidate *args: + import runpy + from pathlib import Path + runpy.run_path(str(Path.cwd() / "tooling/release_candidate.py"), run_name="__main__") + # Require the GitHub release title and body to match the generated notes. [private] _release-notes-check tag: @@ -339,6 +350,11 @@ markdown-ci: markdown-check spell-check markdown-fix: tools-check {{ _run }} research-repo-tools files run {{ _markdown_inputs }} -- rumdl check --fix +# Retry original local receipt retention without measuring or publishing reports. +[positional-arguments] +performance-capture owner *args: + uv run --locked --no-sync python -I tooling/release_candidate.py capture "$@" + # Render the retained release report offline, with independent eligibility checks. performance-doc: python-sync {{ _run }} uv run --locked research-repo-tools performance promote tooling/performance-report.toml @@ -371,14 +387,28 @@ performance-readme current_tag="" current_revision="" baseline_tag="" baseline_r {{ _run }} uv run --locked --no-sync research-repo-tools performance readme tooling/performance-release.toml {{ if current_tag + current_revision + baseline_tag + baseline_revision == "" { "" } else { "--current-release " + quote(current_tag) + " --current-revision " + quote(current_revision) + " --baseline-release " + quote(baseline_tag) + " --baseline-revision " + quote(baseline_revision) } }} # Fetch tags, prepare source assertions, measure, retain, and promote the release report. -performance-release: _ensure-gh python-sync +performance-release progress="human" cadence="300" checkpoint="target/bench-reports/full/checkpoint" mode="fresh": _ensure-gh python-sync #!/usr/bin/env bash set -euo pipefail mkdir -p target/bench-reports/full started=$(date +%s) trap 'status=$?; elapsed=$(($(date +%s) - started)); printf "performance-release: %s seconds; exit %s\\n" "$elapsed" "$status" | tee target/bench-reports/full/runtime.txt' EXIT - uv run --locked --no-sync research-repo-tools performance prepare tooling/performance-release.toml --fetch-tags --allow-git-mutations - {{ _run }} uv run --locked research-repo-tools performance release tooling/performance-release.toml --allow-git-mutations + args=(--checkpoint {{ quote(checkpoint) }}) + case {{ quote(mode) }} in + fresh) + if [[ -e {{ quote(checkpoint) }} ]]; then + echo "Checkpoint exists; use performance-resume or select a new checkpoint." >&2 + exit 2 + fi + uv run --locked --no-sync research-repo-tools performance prepare tooling/performance-release.toml --fetch-tags --allow-git-mutations --release-intent + ;; + resume) args+=(--resume) ;; + *) echo "mode must be fresh or resume" >&2; exit 2 ;; + esac + {{ _run }} uv run --locked --no-sync python -I tooling/release_candidate.py acquire --progress {{ quote(progress) }} --progress-cadence {{ quote(cadence) }} "${args[@]}" + +# Revalidate a checkpoint and acquire only eligible missing work, reusing saved preparation. +performance-resume checkpoint="target/bench-reports/full/checkpoint" progress="human" cadence="300": (performance-release progress cadence checkpoint "resume") # Render a named metric/statistic/axis view from explicitly retained or scratch inputs. plot-vs-linalg current_tag current_revision baseline_tag baseline_revision view="lu_solve-median-linear" input="retained": python-sync @@ -416,9 +446,12 @@ python-sync: _ensure-uv python-typecheck: python-sync uv run --locked research-repo-tools python typecheck +# Derive the already accepted original acquisition without measuring or building. +release-archive owner selection="reviewed": (_release-candidate "archive" owner "--selection" selection) + # Preflight, upload or verify a captured draft target with an isolated trusted runtime. [positional-arguments] -[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.6", "python", "-I")] +[script("uvx", "--python", "3.15", "--from", "research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e", "python", "-I")] release-assets operation tag *args: import sys from pathlib import Path @@ -428,10 +461,23 @@ release-assets operation tag *args: commit = run_git_command(["--no-pager", "rev-parse", "HEAD"], cwd=root, timeout=30).stdout.strip() sys.exit(main(["release", "assets", sys.argv[1], "tooling/release-assets.toml", "--checkout", str(root), "--commit", commit, "--tag", sys.argv[2], *sys.argv[3:]])) +# Verify upstream tooling publication before release preparation or acquisition. +release-dependencies: + uv run --locked --no-sync research-repo-tools release dependencies + +# Bind the original local acquisition to clean HEAD and signed native CI. +release-evidence owner selection="reviewed": (_release-candidate "verify" owner "--selection" selection) + +# Require the release event, reviewed commit, native checks, assets, and notes. +release-gate tag: + uv run --locked --no-sync research-repo-tools release gate {{ quote(tag) }} + gh api "repos/acgetchell/la-stack/releases/$(jq -er '.release.id' "$GITHUB_EVENT_PATH")" | jq -e '.immutable == true' >/dev/null + just _release-notes-check {{ quote(tag) }} + # Verify signed native/Cargo evidence; accept --expected PATH and --github-output PATH. [positional-arguments] -[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.6", "python", "-I")] -release-evidence *args: +[script("uvx", "--python", "3.15", "--from", "research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e", "python", "-I")] +release-native-evidence *args: import sys import tempfile from pathlib import Path @@ -443,12 +489,6 @@ release-evidence *args: with tempfile.TemporaryDirectory(prefix="la-stack-release-evidence-") as temporary: sys.exit(main(["release", "evidence", "verify", "tooling/release-evidence.toml", "--checkout", str(root), "--commit", commit, "--version", read_package_info(root).version, "--output", str(Path(temporary).resolve() / "selection.json"), *sys.argv[1:]])) -# Require the release event, reviewed commit, native checks, assets, and notes. -release-gate tag: - uv run --locked --no-sync research-repo-tools release gate {{ quote(tag) }} - gh api "repos/acgetchell/la-stack/releases/$(jq -er '.release.id' "$GITHUB_EVENT_PATH")" | jq -e '.immutable == true' >/dev/null - just _release-notes-check {{ quote(tag) }} - # Print release notes from the root changelog or a completed minor archive. release-notes tag: python-sync uv run --locked --group dev research-repo-tools changelog notes {{ quote(tag) }} @@ -458,10 +498,21 @@ release-package: {{ _run }} cargo publish --locked --package la-stack --registry crates-io --dry-run {{ _run }} cargo publish --locked --package la-stack --registry crates-io --dry-run --features exact +# Freeze the canonical attached archive ID and bytes before publishing the draft. +release-publication owner selection="reviewed": (_release-candidate "seal" owner "--selection" selection) + # Approve the benchmark-equipped draft as the authenticated maintainer. -release-publish tag: (_release-notes-check tag) release-evidence +release-publish tag owner selection="reviewed": (_release-notes-check tag) (release-evidence owner selection) (release-publication owner selection) uv run --locked --no-sync research-repo-tools release publish {{ quote(tag) }} --approve +# Verify the maintainer's dispatch input; never acquire timings or build packages. +[positional-arguments] +release-receive destination *args: + just _release-candidate receive "$@" + +# Stage accepted inert assets and print the independently retained dispatch record. +release-stage owner selection="reviewed": (_release-candidate "stage" owner "--selection" selection) + # Verify the published GitHub release, required assets, and exact crates.io version. release-verify tag: uv run --locked --no-sync research-repo-tools release verify {{ quote(tag) }} --attempts 7 --interval 10 @@ -559,7 +610,7 @@ test-integration-compile: tools-check # Run Python consumer integration and benchmark-policy tests. test-python: tools-check python-sync - {{ semgrep_stack_limit }} {{ _run }} uv run --locked pytest -q + {{ semgrep_stack_limit }} {{ _run }} uv run --locked pytest -q {{ _pytest_parallel }} --durations=20 # Run all runnable Rust tests and both doctest configurations. test-rust: test-rust-ci test-doc test-doc-exact diff --git a/pyproject.toml b/pyproject.toml index 381a5c6..337ff57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,11 +108,16 @@ override-dependencies = [ "tomli==2.5.0", ] +# Temporary downstream rehearsal pin; return to PyPI after the supplying release. +[tool.uv.sources] +research-repo-tools = { git = "https://github.com/acgetchell/research-repo-tools", rev = "055353c630604b1ae3438c1eaf534ba85e067a7e" } + [dependency-groups] tooling = [ "research-repo-tools[python-tools,semgrep]==0.2.6" ] dev = [ { include-group = "tooling" }, "actionlint-py==1.7.12.25", + "pytest-xdist==3.8.0", "shellcheck-py==0.11.0.1", "shfmt-py==4.2.0", "yamllint==1.38.0", @@ -168,6 +173,7 @@ required-checks = [ required-assets = [ "la-stack-v{version}-criterion-baseline.tar.gz" ] [tool.research-repo-tools.release] +published-dependencies = [ "research-repo-tools" ] # Release preparation and post-merge tagging can happen on different UTC days. date-policy = "declared" tag-policy = "canonical-stable" diff --git a/semgrep.yaml b/semgrep.yaml index b951a5b..e7d9022 100644 --- a/semgrep.yaml +++ b/semgrep.yaml @@ -512,7 +512,6 @@ rules: - "/.github/workflows/release-*.yaml" - "/tests/semgrep/.github/workflows/release-*.yml" - "/tests/semgrep/.github/workflows/release-*.yaml" - - "/.github/actions/prepare-release-benchmarks/action.yml" pattern-either: - patterns: - pattern: | diff --git a/src/lib.rs b/src/lib.rs index 33d0e1a..b20e15f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,11 @@ #![forbid(unsafe_code)] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] -#![doc = include_str!("../README.md")] +#![cfg_attr( + not(doc), + doc = "Small, fixed-dimension linear algebra with checked numerical contracts." +)] +#![cfg_attr(doc, doc = include_str!("../README.md"))] #[cfg(doc)] mod readme_doctests { diff --git a/tests/tooling/test_candidate_transfer.py b/tests/tooling/test_candidate_transfer.py new file mode 100644 index 0000000..97d5f70 --- /dev/null +++ b/tests/tooling/test_candidate_transfer.py @@ -0,0 +1,327 @@ +"""Exercise local ownership and actual hosted handoff code without credentials.""" + +import hashlib +import json +import runpy +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import yaml +from research_repo_tools.release_assets import GitHubRelease, ReleaseTarget, serialize_release_target + +from tooling import release_candidate as candidate + +ROOT = Path(__file__).resolve().parents[2] +SHA = "a" * 40 +TAG = "v0.5.3" + + +@pytest.fixture +def transfer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """Supply explicit independent assertions and provider bytes, never timings.""" + target = ReleaseTarget(candidate.REPOSITORY, TAG, 42, SHA, TAG) + data: dict[str, Any] = { + "schema": 1, + "target": json.loads(serialize_release_target(target)), + "acquisition": "b" * 64, + "publication": "c" * 64, + "assets": {}, + } + payloads = {role: (role + "\r\n").encode() + b"\xff" for role in candidate.ROLES} + for identifier, (role, payload) in enumerate(payloads.items(), 1): + data["assets"][role] = {"name": f"candidate-{role}.bin", "id": identifier, "sha256": hashlib.sha256(payload).hexdigest()} + for key, value in { + "RUNNER_TEMP": str(tmp_path), + "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_REPOSITORY": candidate.REPOSITORY, + "GITHUB_REF": f"refs/tags/{TAG}", + "GITHUB_SHA": SHA, + "RELEASE_TAG": TAG, + "GITHUB_ACTOR": "acgetchell", + "GITHUB_TRIGGERING_ACTOR": "acgetchell", + "GITHUB_WORKFLOW_REF": f"{candidate.REPOSITORY}/.github/workflows/release-benchmarks.yml@refs/tags/{TAG}", + }.items(): + monkeypatch.setenv(key, value) + calls: list[str] = [] + + def release(*_args: object, **_kwargs: object) -> GitHubRelease: + calls.append("identity") + return GitHubRelease( + repository=candidate.REPOSITORY, + tag=TAG, + identifier=data["target"]["release_id"], + draft=True, + immutable=False, + prerelease=False, + assets=tuple((item["name"], item["id"], len(payloads[role]), item["sha256"]) for role, item in data["assets"].items()), + title=TAG, + ) + + def download(_root: Path, _repository: str, _tag: str, name: str, destination: Path, *, expected_sha256: str, **_kwargs: object) -> None: + calls.append("download") + role = next(role for role, item in data["assets"].items() if item["name"] == name) + assert hashlib.sha256(payloads[role]).hexdigest() == expected_sha256 + destination.write_bytes(payloads[role]) + + def upload(_root: Path, _repository: str, _tag: str, path: Path, *, target: ReleaseTarget, **_kwargs: object) -> None: + assert target.release_id == 42 + assert path.read_bytes() == payloads["archive"] + calls.append("upload") + + monkeypatch.setattr("research_repo_tools.release_assets.revalidate_release_target", release) + monkeypatch.setattr("research_repo_tools.release_assets.download_release_asset", download) + monkeypatch.setattr("research_repo_tools.release_assets.publish_release_asset", upload) + return {"data": data, "payloads": payloads, "calls": calls, "target": target, "release": release, "download": download} + + +def workflow_python(tmp_path: Path, job: str) -> Path: + """Run the exact trusted inline block; provider functions are substituted only.""" + workflow = yaml.safe_load((ROOT / ".github/workflows/release-benchmarks.yml").read_bytes()) + step = next(item for item in workflow["jobs"][job]["steps"] if "<<'PY'" in item.get("run", "")) + script = tmp_path / f"{job}.py" + script.write_text(step["run"].split("<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0], encoding="utf-8", newline="\n") + return script + + +@pytest.mark.parametrize("failure", ["none", "ref", "commit", "asset-id", "archive", "acceptance"]) +def test_actual_writer_handoff_rejects_replacement_before_upload( + transfer: dict[str, Any], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, +) -> None: + data, payloads = transfer["data"], transfer["payloads"] + monkeypatch.setenv("CANDIDATE_TRANSFER", json.dumps(data)) + runpy.run_path(str(workflow_python(tmp_path, "fetch")), run_name="__main__") + assert {path.stem.split(".")[0]: path.read_bytes() for path in (tmp_path / "incoming-candidate").iterdir()} == payloads + directory = tmp_path / "accepted-candidate" + directory.mkdir() + (directory / "acceptance.json").write_bytes(payloads["acceptance"]) + (directory / f"la-stack-{TAG}-criterion-baseline.tar.gz").write_bytes(payloads["archive"]) + if failure in {"ref", "commit"}: + monkeypatch.setenv("GITHUB_REF" if failure == "ref" else "GITHUB_SHA", "changed") + elif failure == "asset-id": + data["assets"]["policy"]["id"] += 100 + elif failure in {"archive", "acceptance"}: + path = directory / ("acceptance.json" if failure == "acceptance" else f"la-stack-{TAG}-criterion-baseline.tar.gz") + path.write_bytes(path.read_bytes() + b"changed") + script = workflow_python(tmp_path, "attach") + if failure == "none": + runpy.run_path(str(script), run_name="__main__") + assert transfer["calls"].count("upload") == 1 + else: + with pytest.raises(ValueError, match=r"dispatch differs|asset ID changed|verified artifact differs"): + runpy.run_path(str(script), run_name="__main__") + assert "upload" not in transfer["calls"] + + +@pytest.mark.parametrize("failure", ["duplicate", "missing", "extra", "path", "bool-id", "digest", "alias", "attached"]) +def test_transfer_parser_rejects_invalid_assertions(transfer: dict[str, Any], failure: str) -> None: + data = transfer["data"] + if failure == "missing": + del data["assets"]["archive"] + elif failure == "extra": + data["unexpected"] = True + elif failure == "path": + data["assets"]["archive"]["name"] = "../archive" + elif failure == "bool-id": + data["assets"]["archive"]["id"] = True + elif failure == "digest": + data["acquisition"] = "short" + elif failure == "alias": + data["assets"]["archive"]["id"] = 1 + elif failure == "attached": + data["assets"]["attached"] = {"name": f"la-stack-{TAG}-criterion-baseline.tar.gz", "id": 9, "sha256": "d" * 64} + payload = json.dumps(data) + if failure == "duplicate": + payload = payload.replace('"schema": 1', '"schema": 1, "schema": 1') + diagnostics = { + "duplicate": "duplicate transfer field", + "missing": "four original assets", + "extra": "invalid candidate transfer schema", + "path": "safe names", + "bool-id": "positive exact IDs", + "digest": "full SHA-256", + "alias": "distinct names and IDs", + "attached": "canonical attached archive", + } + with pytest.raises(ValueError, match=diagnostics[failure]): + candidate.parse_transfer(payload.encode()) + + +@pytest.mark.parametrize("key", ["GITHUB_ACTOR", "GITHUB_TRIGGERING_ACTOR", "GITHUB_EVENT_NAME", "GITHUB_WORKFLOW_REF", "GITHUB_SHA", "GITHUB_REF"]) +def test_dispatch_authentication_precedes_provider_access(transfer: dict[str, Any], tmp_path: Path, monkeypatch: pytest.MonkeyPatch, key: str) -> None: + monkeypatch.setenv(key, "untrusted") + with pytest.raises(ValueError, match="maintainer's exact workflow/tag dispatch"): + candidate.receive(tmp_path / "checkout", tmp_path / "output", json.dumps(transfer["data"]).encode(), source=tmp_path) + assert transfer["calls"] == [] + assert not (tmp_path / "output").exists() + + +def test_staging_retry_keeps_original_draft_before_any_upload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, owner = tmp_path / "checkout", tmp_path / "owner" + root.mkdir() + owner.mkdir() + final = candidate.final_directory(owner, SHA, "reviewed") + final.mkdir(parents=True) + for name in ("policy.json", "acceptance.json", "archive.tar.gz"): + (final / name).write_bytes(name.encode()) + for name in ("publication.json", "acquisition.sha256", "publication.sha256"): + (owner / name).write_bytes(b"a" * 64) + monkeypatch.setattr(candidate, "verify", lambda *_args: final / "acceptance.json") + monkeypatch.setattr(candidate, "resolve_revision", lambda *_args: SHA) + monkeypatch.setattr(candidate, "archive", lambda *_args: final / "archive.tar.gz") + monkeypatch.setattr(candidate, "parse_candidate_policy", lambda *_args: SimpleNamespace(tag=TAG)) + original = ReleaseTarget(candidate.REPOSITORY, TAG, 42, SHA, TAG) + preflights: list[int] = [] + + def preflight(*_args: object, **_kwargs: object) -> ReleaseTarget: + preflights.append(42) + return original + + monkeypatch.setattr(candidate, "preflight_release_target", preflight) + monkeypatch.setattr(candidate, "revalidate_release_target", lambda *_args, **_kwargs: None) + + def interrupted(*_args: object, **_kwargs: object) -> None: + assert (final / "target.json").read_bytes() == serialize_release_target(original) + message = "interrupted upload" + raise OSError(message) + + monkeypatch.setattr(candidate, "publish_release_asset", interrupted) + with pytest.raises(OSError, match="interrupted upload"): + candidate.stage(root, owner, "reviewed") + + def replaced(_root: Path, target: ReleaseTarget, **_kwargs: object) -> None: + assert target == original + message = "original draft no longer exists" + raise ValueError(message) + + monkeypatch.setattr(candidate, "revalidate_release_target", replaced) + monkeypatch.setattr(candidate, "publish_release_asset", lambda *_args, **_kwargs: pytest.fail("replacement draft must never receive an upload")) + with pytest.raises(ValueError, match="original draft no longer exists"): + candidate.stage(root, owner, "reviewed") + assert preflights == [42] + + +def test_owner_records_reject_overwrite_and_path_escape(tmp_path: Path) -> None: + candidate.freeze(tmp_path, {"record": b"original"}) + candidate.freeze(tmp_path, {"record": b"original"}) + with pytest.raises(ValueError, match="immutable output differs"): + candidate.freeze(tmp_path, {"record": b"replacement", "other": b"new"}) + with pytest.raises(ValueError, match="within their record"): + candidate.freeze(tmp_path, {"../escaped": b"new"}) + assert (tmp_path / "record").read_bytes() == b"original" + assert not (tmp_path / "other").exists() + + +@pytest.mark.parametrize("failure", ["none", "missing-attached", "changed-attached", "title", "mutable", "changed-acceptance", "reviewed-digest"]) +def test_publication_pins_canonical_archive_and_full_acceptance( + transfer: dict[str, Any], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, +) -> None: + data, payloads = transfer["data"], transfer["payloads"] + data["assets"]["attached"] = {"name": f"la-stack-{TAG}-criterion-baseline.tar.gz", "id": 9, "sha256": data["assets"]["archive"]["sha256"]} + payloads["attached"] = payloads["archive"] + if failure == "missing-attached": + del data["assets"]["attached"] + del payloads["attached"] + assertions = json.dumps(data).encode() + if failure == "changed-attached": + data["assets"]["attached"]["id"] = 10 + if failure == "changed-acceptance": + data["assets"]["acceptance"]["sha256"] = "f" * 64 + root = tmp_path / "checkout" + root.mkdir() + monkeypatch.setenv("GITHUB_WORKFLOW_REF", f"{candidate.REPOSITORY}/.github/workflows/publish-crates.yml@refs/tags/{TAG}") + monkeypatch.setattr(candidate, "check_release_dependencies", lambda *_args: None) + monkeypatch.setattr(candidate, "load_config", lambda **_kwargs: None) + + def published(*_args: object, **_kwargs: object) -> GitHubRelease: + observed = transfer["release"]() + assert isinstance(observed, GitHubRelease) + return replace(observed, draft=False, immutable=failure != "mutable", title="wrong" if failure == "title" else TAG) + + monkeypatch.setattr(candidate, "check_reviewed_release", published) + monkeypatch.setattr(candidate, "download_release_asset", transfer["download"]) + monkeypatch.setattr(candidate, "parse_candidate_policy", lambda *_args: None) + monkeypatch.setattr(candidate, "load_evidence_policy", lambda *_args: None) + monkeypatch.setattr(candidate, "verify_candidate_archive", lambda *_args, **_kwargs: None) + selections: list[bytes | None] = [] + + def verify(*_args: object, expected: bytes | None, **_kwargs: object) -> bytes: + selections.append(expected) + accepted = payloads["acceptance"] + assert isinstance(accepted, bytes) + return accepted + + monkeypatch.setattr(candidate, "verify_candidate_release", verify) + output = tmp_path / "accepted" + digest = "f" * 64 if failure == "reviewed-digest" else hashlib.sha256(payloads["acceptance"]).hexdigest() + if failure == "none": + archive = candidate.receive(root, output, assertions, source=candidate.ReviewedAcceptance(digest)) + assert archive.read_bytes() == payloads["archive"] + assert (output / "acceptance.json").read_bytes() == payloads["acceptance"] + approved = candidate.ReviewedAcceptance(digest, (output / "acceptance.json").read_bytes()) + assert candidate.receive(root, output, assertions, source=approved).read_bytes() == payloads["archive"] + assert selections == [None, payloads["acceptance"]] + else: + diagnostic = { + "missing-attached": "independently selected canonical archive", + "changed-attached": "asset identity changed", + "title": "release identity changed", + "mutable": "must be immutable", + "changed-acceptance": "asset identity changed", + "reviewed-digest": "differs from the independently reviewed digest", + }[failure] + with pytest.raises(ValueError, match=diagnostic): + candidate.receive(root, output, assertions, source=candidate.ReviewedAcceptance(digest)) + assert selections == ([None] if failure == "reviewed-digest" else []) + assert not (output / "acceptance.json").exists() + + +@pytest.mark.parametrize("failure", ["missing", "malformed", "replaced"]) +def test_approval_receipt_requires_independent_identity(failure: str) -> None: + reviewed = b"owner-reviewed full acceptance\r\n" + digest = hashlib.sha256(reviewed).hexdigest() + selected = "" if failure == "missing" else "a" * 63 if failure == "malformed" else digest + expected = reviewed + b"replacement" if failure == "replaced" else None + diagnostic = "approval receipt differs" if failure == "replaced" else "independently reviewed full acceptance" + with pytest.raises(ValueError, match=diagnostic): + candidate.ReviewedAcceptance(selected, expected) + + +def test_explicit_review_selection_preserves_previous_acceptance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, owner = tmp_path / "checkout", tmp_path / "owner" + root.mkdir() + owner.mkdir() + for name in ("policy.json", "acquisition.sha256", "publication.sha256"): + (owner / name).write_bytes(b"a" * 64) + monkeypatch.setattr(candidate, "check_release_dependencies", lambda *_args: None) + monkeypatch.setattr(candidate, "resolve_revision", lambda *_args: SHA) + # Binding behavior is covered by the full synthetic acquisition test. + monkeypatch.setattr(candidate, "parse_candidate_policy", lambda *_args: ReleaseTarget(candidate.REPOSITORY, TAG, 42, SHA, TAG)) + monkeypatch.setattr(candidate, "serialize_candidate_policy", serialize_release_target) + requests: list[list[str]] = [] + + def shared(arguments: list[str]) -> int: + requests.append(arguments) + output = Path(arguments[arguments.index("--output") + 1]) + selected = Path(arguments[arguments.index("--policy") + 1]).parent.name + value = selected.encode() + if "--acceptance" in arguments: + assert Path(arguments[arguments.index("--acceptance") + 1]).read_bytes() == value + output.write_bytes(value) + return 0 + + monkeypatch.setattr(candidate, "shared_main", shared) + first = candidate.verify(root, owner) + assert candidate.verify(root, owner) == first + second = candidate.verify(root, owner, "retry-2") + assert first.read_bytes() == b"reviewed" + assert second.read_bytes() == b"retry-2" + assert ["--acceptance" in arguments for arguments in requests] == [False, True, False] diff --git a/tests/tooling/test_cargo_update_integration.py b/tests/tooling/test_cargo_update_integration.py index a0403d3..b8a0b92 100644 --- a/tests/tooling/test_cargo_update_integration.py +++ b/tests/tooling/test_cargo_update_integration.py @@ -3,6 +3,7 @@ import hashlib import io import json +import select import shutil import sys import tarfile @@ -11,17 +12,44 @@ from functools import partial from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, override import pytest from research_repo_tools.process import run_command if TYPE_CHECKING: from collections.abc import Iterator + from socket import socket REPO_ROOT = Path(__file__).resolve().parents[2] +class RegistryHandler(SimpleHTTPRequestHandler): + """Make HTTP/1.0 connection closure explicit, including empty 304 replies.""" + + @override + def end_headers(self) -> None: + # cargo-edit's HTTP client can otherwise pool a 304 connection before + # this single-request server closes it, racing the next index lookup. + self.send_header("Connection", "close") + super().end_headers() + + +class ClientClosingServer(HTTPServer): + """Expose premature connection reuse without a timing-dependent sleep.""" + + @override + def shutdown_request(self, request: socket | tuple[bytes, socket]) -> None: + assert not isinstance(request, tuple), "the HTTP registry requires a TCP socket" + # Hold the server's close until the client closes or sends another + # request. Reusing this HTTP/1.0 connection then fails deterministically. + # Bound the wait so a broken client cannot hang fixture cleanup. + try: + select.select([request], [], [], 5) + finally: + super().shutdown_request(request) + + def crate(registry: Path, name: str, version: str) -> dict[str, object]: """Create one deterministic registry candidate for native Cargo resolution.""" buffer = io.BytesIO() @@ -42,7 +70,7 @@ def crate(registry: Path, name: str, version: str) -> dict[str, object]: @pytest.fixture -def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: +def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, server_type: type[HTTPServer]) -> Iterator[Path]: """Keep actual tool declarations and recipes; substitute only registry inputs.""" root = tmp_path / "cargo consumer" root.mkdir() @@ -58,7 +86,7 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: versions = {name: dependencies[name]["version"] for name in ("num-bigint", "num-rational", "num-traits")} registry = tmp_path / "registry" registry.mkdir() - server = HTTPServer(("127.0.0.1", 0), partial(SimpleHTTPRequestHandler, directory=str(registry))) + server = server_type(("127.0.0.1", 0), partial(RegistryHandler, directory=str(registry))) host = f"http://127.0.0.1:{server.server_port}" index = registry / "index" index.mkdir() @@ -89,6 +117,7 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: @pytest.mark.parametrize("recipe", ["update-cargo-dependencies", "update-dependencies"]) +@pytest.mark.parametrize("server_type", [HTTPServer, ClientClosingServer], ids=["normal-close", "client-close"]) def test_native_updates_advance_eligible_dependencies_and_preserve_coupled_requirements(consumer: Path, recipe: str) -> None: """Cargo upgrade and cargo update run through the installed checked toolchain.""" before = tomllib.loads((consumer / "Cargo.toml").read_text(encoding="utf-8"))["dependencies"] @@ -97,8 +126,9 @@ def test_native_updates_advance_eligible_dependencies_and_preserve_coupled_requi ["--justfile", str(consumer / "recipes.just"), "--working-directory", str(consumer), recipe], cwd=consumer, timeout=120, + check=False, ) - assert result.returncode == 0 + assert result.returncode == 0, result.stdout + result.stderr after = tomllib.loads((consumer / "Cargo.toml").read_text(encoding="utf-8"))["dependencies"] assert after["num-bigint"] == before["num-bigint"] assert after["num-rational"] == before["num-rational"] diff --git a/tests/tooling/test_performance_workflow.py b/tests/tooling/test_performance_workflow.py index 904d7ec..7775172 100644 --- a/tests/tooling/test_performance_workflow.py +++ b/tests/tooling/test_performance_workflow.py @@ -1,38 +1,55 @@ """Installed workflow acceptance with independent la-stack scientific declarations.""" +import csv import fnmatch import hashlib +import io import json import os import shlex import shutil +import subprocess import sys import tomllib +from dataclasses import replace from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast import pytest import yaml -from research_repo_tools.common_measurement import CompleteMeasurementPlan, DiscoveredSeries, DiscoveryPolicy, measure_prepared_pair, validate_run_policy -from research_repo_tools.complete_runs import RunSeries -from research_repo_tools.evidence import load_evidence, publish_evidence +from research_repo_tools.candidate_reuse import parse_candidate_policy +from research_repo_tools.cli import main +from research_repo_tools.common_measurement import CompleteMeasurementPlan, DiscoveryPolicy, validate_run_policy +from research_repo_tools.evidence import deterministic_json, load_evidence, publish_evidence from research_repo_tools.just_inspect import dry_run, inspect_justfile -from research_repo_tools.measurement import load_measurement +from research_repo_tools.measurement import load_measurement, measure_pair +from research_repo_tools.performance_progress import ProgressReporter from research_repo_tools.performance_release import Preparation, serialize_preparation -from research_repo_tools.process import run_command +from research_repo_tools.performance_reports import load_report_plan +from research_repo_tools.process import run_command, run_command_live +from research_repo_tools.publication import publish_publication +from research_repo_tools.release_assets import GitHubRelease, ReleaseTarget, serialize_release_target from research_repo_tools.release_pairs import ReleasePair +from research_repo_tools.scheduled_runs import ScheduledRun, select_trial +from research_repo_tools.suite_measurement import SuiteMeasurementPlan +from research_repo_tools.suite_runs import comparison_incompatibilities, parse_suite_run from research_repo_tools.worktrees import capture_snapshot +from tooling.release_candidate import ReviewedAcceptance, archive, capture, freeze, receive, verify + if TYPE_CHECKING: - import subprocess + from collections.abc import Sequence from research_repo_tools.evidence import Evidence ROOT = Path(__file__).resolve().parents[2] -PAIR = ReleasePair("v0.5.1", "v0.5.0") +PAIR = ReleasePair("v0.5.2", "v0.5.1") DRIVER = """import json, os, sys from pathlib import Path phase, suite, action = sys.argv[1:] +if phase == "source": + phase = Path("src/lib.rs").read_text(encoding="utf-8").strip().removeprefix("// ").split()[0] +assert suite in IDS[phase], "unsupported API was compiled/invoked" if action == "gate": Path("target").mkdir(exist_ok=True) with Path("target/gates.txt").open("a", encoding="utf-8", newline="\\n") as stream: @@ -55,15 +72,16 @@ def estimate(value): "benchmark.json": {"full_id": name}, "estimates.json": {"mean": estimate(mean), "median": estimate(median)}, "sample.json": {"iters": [1] * 100, "times": [median] * 100}, + "tukey.json": [median * .5, median * .75, median * 1.25, median * 1.5], } for filename, value in data.items(): (path / filename).write_text(json.dumps(value), encoding="utf-8", newline="\\n") """ -def declaration(root: Path, view: str = "full") -> CompleteMeasurementPlan: +def declaration(root: Path, view: str = "full") -> CompleteMeasurementPlan | SuiteMeasurementPlan: plan = load_measurement(root, f"tooling/measurements/{view}.toml") - assert isinstance(plan, CompleteMeasurementPlan) + assert isinstance(plan, (CompleteMeasurementPlan, SuiteMeasurementPlan)) return plan @@ -85,13 +103,36 @@ def scientific_ids(view: str = "full") -> tuple[set[str], set[str]]: return library, peers -def baseline_ids(view: str = "full") -> set[str]: - """Older releases cannot supply timings for APIs introduced in v0.5.2.""" +def baseline_ids(view: str = "full", release: str = PAIR.baseline) -> set[str]: + """Derive historical expectations from the independent scientific API catalog.""" library, _peers = scientific_ids(view) if view != "full": return library catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) - return {name for name in library if not any(fnmatch.fnmatchcase(name, pattern) for pattern in catalog["baseline"]["current-only"])} + version = tuple(map(int, release.removeprefix("v").split("."))) + return { + name + for name in library + if not any( + fnmatch.fnmatchcase(name, pattern) and version < tuple(map(int, since.removeprefix("v").split("."))) + for pattern, since in catalog["availability"].items() + ) + } + + +def suite_ids() -> dict[str, set[str]]: + """Partition the scientific catalog independently of the acquisition declaration.""" + catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) + library, peers = scientific_ids() + families = { + "column-basis": {name for name in library if name.startswith("column_basis/")}, + "interval-solve": {name for name in library if name.startswith(("interval_division/", "interval_solve_d"))}, + "polynomial": {name for name in library if name.startswith("polynomial/")}, + "rational-dot": {name for name in library if name.startswith("rational_dot/")}, + "lu-solve": {name for name in library if name.endswith("/la_stack_lu_solve")} | peers, + } + separate = set().union(*families.values()) + return {"core-default": set(catalog["release"]["default"]) - separate, "core-exact": set(catalog["release"]["exact"]) - separate, **families} def synthetic_timings() -> dict[str, dict[str, dict[str, float]]]: @@ -107,64 +148,78 @@ def synthetic_timings() -> dict[str, dict[str, dict[str, float]]]: } -@pytest.mark.parametrize("view", ["full", "non-exact"]) -def test_declared_scientific_sampling_coverage_and_original_peer_phases(view: str) -> None: - library, peers = scientific_ids(view) - plan = declaration(ROOT, view) - catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) - assert plan.measurement.minimum_release == "v" + ".".join(str(part) for part in catalog["minimum-release"]) - baseline = baseline_ids(view) - for phase, required in ((plan.baseline, baseline | peers), (plan.current, library)): +@pytest.mark.parametrize("release", ["v0.5.0", "v0.5.1", "v0.5.2", "v0.5.3"]) +def test_declared_scientific_sampling_coverage_and_original_peer_phases(release: str) -> None: + plan = declaration(ROOT) + assert isinstance(plan, SuiteMeasurementPlan) + expected = suite_ids() + library, peers = scientific_ids() + assert len(library) == 1403 + assert len(peers) == 16 + assert set().union(*expected.values()) == library | peers + assert sum(map(len, expected.values())) == len(library | peers) + available: set[str] = set() + for suite in plan.suites: + assert suite.nominal_seconds_per_case == 8 + assert suite.measurement.baseline.command == suite.measurement.current.command + assert suite.measurement.current.gate == (*suite.measurement.baseline.gate[:-1], suite.measurement.baseline.gate[-1] + "-current") + assert suite.measurement.baseline.sources == ("src/**/*.rs", "Cargo.toml", "README.md") + assert "README.md" not in suite.measurement.sources + assert dict(suite.measurement.measurement.context)["acquisition-owner"] == "local" + for phase in (suite.measurement.baseline, suite.measurement.current): + assert isinstance(phase.policy, DiscoveryPolicy) + assert set(phase.policy.discovery.required) == expected[suite.name] + assert phase.policy.sample_count == 100 + assert phase.policy.statistics == ("mean", "median") + assert phase.policy.confidence_level == 0.95 + assert phase.policy.unit == "ns" + environment = dict(phase.environment) + assert environment["RUSTFLAGS"] == environment["CARGO_ENCODED_RUSTFLAGS"] == "--cap-lints=warn" + assert environment["CARGO_TARGET_DIR"] == "target/build" + assert environment["CARGO_BUILD_BUILD_DIR"] == "target/build/intermediates" + assert environment["RUSTUP_TOOLCHAIN"] == tomllib.loads((ROOT / "rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"] + if suite.decision(release, "{}")["state"] == "available": + available.update(expected[suite.name]) + assert {"context.os", "context.architecture", "context.cpu", "harness_sha256"} <= set(suite.measurement.measurement.compatible) + assert available == baseline_ids(release=release) | peers + lu = next(suite for suite in plan.suites if suite.name == "lu-solve") + assert {item.name: item.phase for item in lu.measurement.series} == { + "la-stack baseline": "baseline", + "la-stack current": "current", + "nalgebra": "current", + "faer": "current", + } + assert { + "benches/**/*.rs", + "tests/exact_bench_config.rs", + "tooling/performance.just", + "tooling/measurements/*.toml", + "pyproject.toml", + "uv.lock", + "dependency-sources.toml", + ".python-version", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + } <= set(plan.harness) + + +def test_non_exact_diagnostic_keeps_original_peer_phases() -> None: + plan = declaration(ROOT, "non-exact") + assert isinstance(plan, CompleteMeasurementPlan) + library, peers = scientific_ids("non-exact") + for phase, required in ((plan.baseline, library | peers), (plan.current, library)): assert isinstance(phase.policy, DiscoveryPolicy) assert set(phase.policy.discovery.required) == required - assert phase.policy.sample_count == 100 - assert phase.policy.statistics == ("mean", "median") - assert phase.policy.confidence_level == 0.95 - assert phase.policy.unit == "ns" - assert phase.gate[-1] == ("gate-public" if view == "full" and phase is plan.current else "gate") - environment = dict(phase.environment) - assert environment["RUSTFLAGS"] == environment["CARGO_ENCODED_RUSTFLAGS"] == "--cap-lints=warn" - assert environment["RUSTUP_TOOLCHAIN"] == tomllib.loads((ROOT / "rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"] assert {item.name: item.phase for item in plan.series} == { "la-stack baseline": "baseline", "la-stack current": "current", "nalgebra": "baseline", "faer": "baseline", } - for item in plan.series: - if isinstance(item, DiscoveredSeries): - assert view == "full" - assert item.name.startswith("la-stack") - measured = baseline if item.phase == "baseline" else library - expected = tuple(sorted(measured | (peers if item.phase == "baseline" else set()))) - assert dict(item.resolve(expected).rows) == {name: name for name in measured} - continue - assert isinstance(item, RunSeries) - if item.name.startswith("la-stack"): - assert dict(item.rows) == {name: name for name in library} - else: - assert dict(item.rows) == { - f"d{dimension}/{name}": f"d{dimension}/{peer}" - for dimension in catalog["dimensions"] - for name, peer_names in catalog["peers"].items() - if view == "non-exact" or name == "la_stack_lu_solve" - for peer in peer_names - if peer.startswith(f"{item.name}_") - } - assert {"context.os", "context.architecture", "context.cpu", "harness_sha256"} <= set(plan.measurement.compatible) - assert {"benches/**/*.rs", "tests/exact_bench_config.rs", "tooling/performance.just", "tooling/measurements/*.toml"} <= set(plan.harness) - if view == "full": - assert { - "pyproject.toml", - "uv.lock", - ".python-version", - "tests/column_basis.rs", - "tests/rational_dot.rs", - "tests/interval_solve.rs", - "tests/common/interval_solve_oracle.rs", - "tests/common/proptest_config.rs", - "tests/polynomial_bench_inputs.rs", - } <= set(plan.harness) def test_routine_ci_validates_benchmarks_without_acquiring_timings() -> None: @@ -210,12 +265,11 @@ def test_routine_ci_validates_benchmarks_without_acquiring_timings() -> None: [("column_basis/*", 55), ("interval_division/*", 5), ("interval_solve_d*/*", 30), ("polynomial/*", 5), ("rational_dot/*", 54)], ) def test_new_api_coverage_is_complete_without_inventing_baseline_timings(pattern: str, count: int) -> None: - plan = declaration(ROOT) - assert isinstance(plan.current.policy, DiscoveryPolicy) - assert isinstance(plan.baseline.policy, DiscoveryPolicy) - current = {name for name in plan.current.policy.discovery.required if fnmatch.fnmatchcase(name, pattern)} + library, _peers = scientific_ids() + current = {name for name in library if fnmatch.fnmatchcase(name, pattern)} assert len(current) == count - assert current.isdisjoint(plan.baseline.policy.discovery.required) + assert current.isdisjoint(baseline_ids(release="v0.5.1")) + assert current <= baseline_ids(release="v0.5.2") def test_named_views_keep_every_metric_statistic_axis_and_explicit_input_choice() -> None: @@ -232,7 +286,9 @@ def test_named_views_keep_every_metric_statistic_axis_and_explicit_input_choice( assert view["plot"]["y-scale"] == axis assert axis in {"linear", "log"} assert [row["coordinate"] for row in view["rows"]] == catalog["dimensions"] - assert [series["name"] for series in view["series"]] == ["la-stack current", "nalgebra", "faer"] + assert [series["name"] for series in view["series"]] == [ + ("" if source == "non-exact" else "lu-solve/") + name for name in ("la-stack current", "nalgebra", "faer") + ] for series, benchmark in zip(view["series"], catalog["metrics"][metric]["benchmarks"], strict=True): assert series["rows"] == {str(d): f"d{d}/{benchmark}" for d in catalog["dimensions"]} measurement = "non-exact" if source == "non-exact" else "full" @@ -250,16 +306,109 @@ def test_canonical_readme_selects_all_lu_solve_medians_and_peer_over_current_rat catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) view = tomllib.loads((ROOT / "tooling/performance-readme.toml").read_text(encoding="utf-8")) assert view["measurement"] == "tooling/measurements/full.toml" + assert view["trial"] == 1 assert view["statistic"] == "median" assert view["unit"] == "ns" assert view["rows"] == [{"key": str(d), "label": str(d), "coordinate": d} for d in catalog["dimensions"]] - assert [series["name"] for series in view["series"]] == ["la-stack current", "nalgebra", "faer"] + assert [series["name"] for series in view["series"]] == ["lu-solve/la-stack current", "lu-solve/nalgebra", "lu-solve/faer"] for series, benchmark in zip(view["series"], catalog["metrics"]["lu_solve"]["benchmarks"], strict=True): assert series["rows"] == {str(d): f"d{d}/{benchmark}" for d in catalog["dimensions"]} assert view["comparisons"] == [ - {"numerator": "nalgebra", "denominator": "la-stack current"}, - {"numerator": "faer", "denominator": "la-stack current"}, + {"numerator": "lu-solve/nalgebra", "denominator": "lu-solve/la-stack current"}, + {"numerator": "lu-solve/faer", "denominator": "lu-solve/la-stack current"}, + ] + + +@pytest.mark.parametrize( + ("failure", "diagnostic"), + [ + ("none", ""), + ("readme", "current compilation reads README.md"), + ("case-alias", "current compilation reads README.md"), + ("missing-bench", "missing benchmark-discovery or correctness-gate"), + ("missing-gate", "missing benchmark-discovery or correctness-gate"), + ("empty", "missing library input"), + ], +) +def test_current_input_gate_requires_both_native_profiles(tmp_path: Path, failure: str, diagnostic: str) -> None: + """Exercise the actual native-Just input guard without compiling or measuring.""" + for name in ("pyproject.toml", "uv.lock", ".python-version"): + shutil.copyfile(ROOT / name, tmp_path / name) + for profile in ("debug", "release"): + if (failure, profile) in {("missing-bench", "release"), ("missing-gate", "debug")}: + continue + path = tmp_path / "target/build" / profile / "deps/la_stack-abcdef.d" + path.parent.mkdir(parents=True) + data = b"target: src/lib.rs\n" + if profile == "release": + if failure == "readme": + data += b"src/../README.md:\n" + elif failure == "case-alias": + data += b"C:\\fixture\\src\\..\\ReadMe.MD:\r\n" + elif failure == "empty": + data = b"" + path.write_bytes(data) + result = run_command( + "just", + ["--no-deps", "--justfile", str(ROOT / "tooling/performance.just"), "--working-directory", str(tmp_path), "_readme-input-check"], + cwd=tmp_path, + env={**os.environ, "UV_PROJECT_ENVIRONMENT": sys.prefix}, + check=False, + ) + assert result.returncode == (0 if failure == "none" else 1), result.stderr + if diagnostic: + assert diagnostic in result.stderr + + +def test_checkpoint_resume_preserves_completed_scientific_pairs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Recover the real LU/column-basis declarations without repeating LU samples.""" + pair = ReleasePair("v0.5.3", "v0.5.2") + _baseline, root = prepared(tmp_path, pair) + full = declaration(root) + assert isinstance(full, SuiteMeasurementPlan) + selected = replace(full, suites=tuple(suite for suite in full.suites if suite.name in {"lu-solve", "column-basis"})) + checkpoint = root / "target/checkpoint" + checkpoint.parent.mkdir() + calls: list[tuple[str, str]] = [] + interrupted = False + + def native(command: str, args: Sequence[str] = (), **kwargs: Any) -> Any: + nonlocal interrupted + if args and args[-1] in {"lu-solve", "column-basis"}: + phase = (cast("Path", kwargs["cwd"]) / "src/lib.rs").read_text(encoding="utf-8").strip().removeprefix("// ") + if (args[-1], phase) == ("column-basis", "current") and not interrupted: + interrupted = True + raise KeyboardInterrupt + calls.append((args[-1], phase)) + return run_command_live(command, args, **kwargs) + + monkeypatch.setattr("research_repo_tools.common_measurement.run_command_live", native) + with pytest.raises(KeyboardInterrupt): + measure_pair(root, selected, pair, working_tree=True, checkpoint=checkpoint, progress=ProgressReporter(mode="none")) + original = (checkpoint / "events.jsonl").read_bytes() + assert calls == [("lu-solve", "baseline"), ("lu-solve", "current"), ("column-basis", "baseline")] + evidence = measure_pair(root, selected, pair, working_tree=True, checkpoint=checkpoint, resume=True, progress=ProgressReporter(mode="none")) + validate_run_policy(evidence, selected, release=True) + assert (checkpoint / "events.jsonl").read_bytes().startswith(original) + assert calls == [ + ("lu-solve", "baseline"), + ("lu-solve", "current"), + ("column-basis", "baseline"), + ("column-basis", "baseline"), + ("column-basis", "current"), ] + trial = select_trial(evidence, 1) + for dimension in (2, 3, 4, 5, 8, 16, 32, 64): + name = f"d{dimension}/la_stack_lu_solve" + assert comparison_incompatibilities(trial, ("baseline", name), ("current", name)) == () + # Input drift rejects before another native measurement and preserves the journal. + completed = (checkpoint / "events.jsonl").read_bytes() + before_calls = list(calls) + (root / "src/lib.rs").write_text("// changed current input\n", encoding="utf-8", newline="\n") + with pytest.raises(ValueError, match="identity changed"): + measure_pair(root, selected, pair, working_tree=True, checkpoint=checkpoint, resume=True, progress=ProgressReporter(mode="none")) + assert calls == before_calls + assert (checkpoint / "events.jsonl").read_bytes() == completed def recipe(root: Path, name: str, *arguments: str) -> subprocess.CompletedProcess[str]: @@ -277,54 +426,52 @@ def recipe(root: Path, name: str, *arguments: str) -> subprocess.CompletedProces def test_native_recipes_preserve_scientific_gates_and_peer_reuse() -> None: justfile = Path("tooling/performance.just") - baseline = dry_run(ROOT, "baseline-all", justfile=justfile).stderr - current = dry_run(ROOT, "current-all", justfile=justfile).stderr - assert baseline.count("cargo bench --locked") == 10 - assert current.count("cargo bench --locked") == 13 - assert "(nalgebra|faer)_lu_solve$" in baseline - assert "(nalgebra|faer)_lu_solve$" not in current - assert "--target" in baseline - assert "--target" in current - assert dry_run(ROOT, "baseline-all", ["list"], justfile=justfile).stderr.count("args+=(--list)") == 10 - assert "--test vs_linalg_inputs --test exact_bench_config" in dry_run(ROOT, "gate", justfile=justfile).stderr - gate = dry_run(ROOT, "gate-public", justfile=justfile).stderr - hosted_gate = dry_run(ROOT, "test-bench-inputs").stderr - for target in ("vs_linalg_inputs", "exact_bench_config", "column_basis", "rational_dot", "interval_solve", "polynomial_bench_inputs"): - assert f"--test {target}" in gate - assert f"--test {target}" in hosted_gate - # New native targets must be considered for release coverage. Diagnostic - # controls are filtered within targets rather than omitting public targets. + plan = declaration(ROOT) + assert isinstance(plan, SuiteMeasurementPlan) + measured_targets: set[str] = set() + for suite in plan.suites: + command = suite.measurement.current.command[-1] + measured = dry_run(ROOT, command, justfile=justfile).stderr + discovered = dry_run(ROOT, command, ["list"], justfile=justfile).stderr + assert measured.count("cargo bench --locked") == discovered.count("args+=(--list)") + assert "--target" in measured + for manifest in (ROOT / "Cargo.toml", ROOT / "benches/comparison/Cargo.toml"): + for target in tomllib.loads(manifest.read_text(encoding="utf-8"))["bench"]: + if f"--bench '{target['name']}'" in measured: + measured_targets.add(target["name"]) + gate = dry_run(ROOT, suite.measurement.current.gate[-1], justfile=justfile).stderr + assert "cargo test --locked" in gate + if suite.since: + assert suite.name.replace("-", "_") in gate or "polynomial_bench_inputs" in gate targets = { bench["name"] for manifest in (ROOT / "Cargo.toml", ROOT / "benches/comparison/Cargo.toml") for bench in tomllib.loads(manifest.read_text(encoding="utf-8"))["bench"] } - current_only = {"column_basis", "interval_solve", "polynomial", "rational_dot"} - for target in targets: - assert (f"--bench '{target}'" in baseline) == (target not in current_only) - assert f"--bench '{target}'" in current + assert measured_targets == targets + core = dry_run(ROOT, "core-default", justfile=justfile).stderr + assert "|lu|lu_pivoting|lu_ill_conditioned|" in core + assert "la_stack_lu_solve" not in core + lu = dry_run(ROOT, "lu-solve", justfile=justfile).stderr + assert "(la_stack|nalgebra|faer)_lu_solve$" in lu + assert lu.count("cargo bench --locked") == 1 for name in ("performance-local", "performance-local-non-exact"): command = dry_run(ROOT, name).stderr assert "research-repo-tools performance measure" in command assert "--mode current-vs-latest" in command - assert "--allow-git-mutations" in command command = dry_run(ROOT, "performance-release").stderr - assert command.count("performance release tooling/performance-release.toml") == 1 - plan = tomllib.loads((ROOT / "tooling/performance-release.toml").read_text(encoding="utf-8")) - assert plan["payload"] == "target/bench-reports/full/run.json" - assert "scripts/" not in command + assert command.count("tooling/release_candidate.py acquire") == 1 + assert command.count("--release-intent") == 1 -def prepared(tmp_path: Path) -> tuple[Path, Path]: +def prepared(tmp_path: Path, pair: ReleasePair = PAIR) -> tuple[Path, Path]: """Replace only native timing/gates with a bounded fixture, retaining real policy.""" - library, peers = scientific_ids() - catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) + _library, peers = scientific_ids() diagnostic_library, diagnostic_peers = scientific_ids("non-exact") ids = { side: { - "public-default": sorted(set(catalog["release"]["default"]) & (baseline_ids() if side == "baseline" else library)), - "public-exact": sorted(set(catalog["release"]["exact"]) & (baseline_ids() if side == "baseline" else library)), - "peers": sorted(peers), + **{name: sorted(cases) for name, cases in suite_ids().items() if side == "current" or cases <= baseline_ids(release=pair.baseline) | peers}, + "all": [], "vs_linalg": sorted(diagnostic_library | (diagnostic_peers if side == "baseline" else set())), } for side in ("baseline", "current") @@ -334,6 +481,9 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: root.mkdir(parents=True) for name in ("pyproject.toml", "uv.lock", ".python-version", "justfile"): shutil.copyfile(ROOT / name, root / name) + # Retain every runtime/configuration input. Operational guides link to + # the full repository, which this bounded generated-README fixture lacks. + shutil.copytree(ROOT / "tooling", root / "tooling", ignore=shutil.ignore_patterns("__pycache__", "*.md")) for pattern in declaration(ROOT).harness: for source in ROOT.glob(pattern): destination = root / source.relative_to(ROOT) @@ -341,24 +491,28 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: shutil.copyfile(source, destination) (root / "src").mkdir() (root / "src/lib.rs").write_text(f"// {phase}\n", encoding="utf-8", newline="\n") + (root / "README.md").write_text("# Fixture\n\n\nOld table\n\n", encoding="utf-8", newline="\n") (root / ".gitignore").write_text("/target/\n", encoding="utf-8", newline="\n") - if phase == "current": - (root / "src/untracked.rs").write_bytes(b"// included working-tree input\n") manifest = root / "Cargo.toml" contents = manifest.read_text(encoding="utf-8") version = tomllib.loads(contents)["package"]["version"] - manifest.write_text(contents.replace(f'version = "{version}"', 'version = "0.5.1"', 1), encoding="utf-8", newline="\n") + manifest.write_text( + contents.replace(f'version = "{version}"', f'version = "{(pair.baseline if phase == "baseline" else pair.current).removeprefix("v")}"', 1), + encoding="utf-8", + newline="\n", + ) (root / "tooling/fixture.py").write_text(f"IDS = {ids!r}\nTIMINGS = {synthetic_timings()!r}\n" + DRIVER, encoding="utf-8", newline="\n") driver = f"{shlex.quote(sys.executable)} -X utf8 tooling/fixture.py" native = 'export CRITERION_HOME := absolute_path(env("CRITERION_HOME", "target/criterion"))\n' - native += '\nbaseline-all mode="measure": (baseline-public-default mode) (baseline-public-exact mode) (baseline-peers mode)\n' - native += '\ncurrent-all mode="measure": (current-public-default mode) (current-public-exact mode)\n' + plan = declaration(ROOT) + assert isinstance(plan, SuiteMeasurementPlan) + for suite in plan.suites: + command = suite.measurement.current.command[-1] + native += f'\n{command} mode="measure":\n {driver} source {suite.name} {{{{mode}}}}\n' for side in ("baseline", "current"): - for suite in ("public-default", "public-exact", "peers"): - native += f'\n{side}-{suite} mode="measure":\n {driver} {side} {suite} {{{{mode}}}}\n' native += f'\n{side}-vs_linalg mode="measure":\n {driver} {side} vs_linalg {{{{mode}}}}\n' - native += f"\ngate:\n {driver} current all gate\n" - native += "\ngate-public: gate\n" + for gate in sorted({phase.gate[-1] for suite in plan.suites for phase in (suite.measurement.baseline, suite.measurement.current)}): + native += f"\n{gate}:\n {driver} source all gate\n" (root / "tooling/performance.just").write_text(native, encoding="utf-8", newline="\n") for config in (root / "tooling/measurements").glob("*.toml"): config.write_text( @@ -367,6 +521,7 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: newline="\n", ) run_command("git", ["init", "--quiet"], cwd=root) + run_command("git", ["add", "."], cwd=root) run_command( "git", [ @@ -384,15 +539,16 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: ], cwd=root, ) - return tmp_path / "baseline", tmp_path / "current" + baseline, current = tmp_path / "baseline", tmp_path / "current" + run_command("git", ["fetch", str(baseline), "HEAD"], cwd=current) + run_command("git", ["-c", "tag.gpgsign=false", "tag", pair.baseline, "FETCH_HEAD"], cwd=current) + (current / "src/untracked.rs").write_bytes(b"// included working-tree input\n") + return baseline, current @pytest.fixture(scope="module") def measured_run(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Evidence]: baseline, current = prepared(tmp_path_factory.mktemp("performance")) - # Retain the independently captured baseline commit for preparation freshness. - run_command("git", ["fetch", str(baseline), "HEAD"], cwd=current) - run_command("git", ["-c", "tag.gpgsign=false", "tag", PAIR.baseline, "FETCH_HEAD"], cwd=current) asserted = [ PAIR.current, run_command("git", ["--no-pager", "rev-parse", "HEAD"], cwd=current).stdout.strip(), @@ -400,15 +556,156 @@ def measured_run(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Eviden run_command("git", ["--no-pager", "rev-parse", "HEAD"], cwd=baseline).stdout.strip(), ] (current / "fixture-identities.json").write_text(json.dumps(asserted), encoding="utf-8", newline="\n") - evidence = measure_prepared_pair(baseline, current, declaration(current), PAIR, working_tree=True) + progress = io.StringIO() + events: list[dict[str, object]] = [] + evidence = measure_pair( + current, + declaration(current), + PAIR, + working_tree=True, + progress=ProgressReporter(stream=progress, observer=events.append), + ) + (current / "target").mkdir(exist_ok=True) + (current / "target/progress.json").write_text(json.dumps(events), encoding="utf-8", newline="\n") + (current / "target/progress.txt").write_text(progress.getvalue(), encoding="utf-8", newline="\n") for path in (ROOT / "tooling").glob("performance-*.toml"): shutil.copyfile(path, current / "tooling" / path.name) - shutil.copytree(ROOT / "tooling/plots", current / "tooling/plots") + shutil.copytree(ROOT / "tooling/plots", current / "tooling/plots", dirs_exist_ok=True) (current / "README.md").write_text("# Fixture\n\n\nOld table\n\n", encoding="utf-8", newline="\n") publish_evidence(evidence, current / "target/bench-reports/full/run.json", current / "target/bench-reports/full/evidence.json") return current, evidence +def test_shared_progress_reports_validated_inventory_and_unknown_eta(measured_run: tuple[Path, Evidence]) -> None: + """Exercise the installed reporter with the real consumer plan and captured logs.""" + root, evidence = measured_run + events = json.loads((root / "target/progress.json").read_text(encoding="utf-8")) + budget = next(event for event in events if event["event"] == "budget") + library, peers = scientific_ids() + baseline = baseline_ids() | peers + total = len(baseline) + len(library) + total += len(peers) + assert total == 2689 + assert budget["total"] == budget["remaining"] == budget["unknown_cases"] == total + assert budget["completed"] == 0 + assert budget["eta_seconds"] is None + assert budget["nominal_seconds"] == total * 8 + assert sum(row["total"] for row in budget["inventory"]) == total + accepted = [event for event in events if event["event"] == "validated"] + assert accepted[-1]["completed"] == total + assert accepted[-1]["remaining"] == 0 + assert all(event["completed"] + event["remaining"] == total for event in accepted) + assert accepted[-1]["eta_seconds"] == 0 + assert {event["state"] for event in events} >= {"discovery", "gate", "acquisition"} + assert all(event["schema"] == "research-repo-tools/acquisition-progress/v1" for event in events) + log = (root / "target/progress.txt").read_text(encoding="utf-8") + assert "ETA unknown" in log + assert f"{total} validated, 0 remaining" in log + assert "\x1b" not in log + assert "\r" not in log + validate_run_policy(evidence, declaration(root)) + trial = select_trial(evidence, 1) + for dimension in (2, 3, 4, 5, 8, 16, 32, 64): + library_id = f"d{dimension}/la_stack_lu_solve" + assert comparison_incompatibilities(trial, ("baseline", library_id), ("current", library_id)) == () + for peer in ("nalgebra", "faer"): + assert comparison_incompatibilities(trial, ("current", f"d{dimension}/{peer}_lu_solve"), ("current", library_id)) == () + + +@pytest.mark.parametrize("baseline", ["v0.5.0", "v0.5.2", "v0.5.3"]) +def test_api_availability_uses_original_source_receipts(tmp_path: Path, baseline: str) -> None: + """The common harness must not change which original revision supports an API.""" + pair = ReleasePair("v0.5.4", baseline) + _baseline, current = prepared(tmp_path, pair) + plan = declaration(current) + assert isinstance(plan, SuiteMeasurementPlan) + # Shared suites have complete-inventory coverage above. Isolate all four + # introduction boundaries here while retaining every one of their 149 cases. + selected = replace(plan, suites=tuple(suite for suite in plan.suites if suite.since)) + evidence = measure_pair(current, selected, pair, working_tree=True, progress=ProgressReporter(mode="none")) + run = validate_run_policy(evidence, selected) + assert isinstance(run, ScheduledRun) + phases = dict(parse_suite_run(select_trial(evidence, 1)).phases) + introduced = scientific_ids()[0] - baseline_ids(release="v0.5.1") + assert len(introduced) == 149 + assert set(phases["current"].policy.expected) == introduced + assert set(phases["baseline"].policy.expected) == (set() if baseline == "v0.5.0" else introduced) + + +@pytest.mark.parametrize("phase", ["baseline", "current"]) +def test_full_plan_rejects_unsupported_release_before_native_commands(tmp_path: Path, phase: str) -> None: + """Schema 5 captures source receipts before enforcing the native release floor.""" + pair = ReleasePair("v0.4.6" if phase == "current" else PAIR.current, "v0.4.6" if phase == "baseline" else PAIR.baseline) + _baseline, current = prepared(tmp_path, pair) + events: list[dict[str, object]] = [] + with pytest.raises(ValueError, match=r"requires v0\.5\.0 or newer"): + measure_pair(current, declaration(current), pair, working_tree=True, progress=ProgressReporter(mode="none", observer=events.append)) + assert not any(event["state"] in {"discovery", "gate", "acquisition"} for event in events) + + +def test_supported_new_api_discovery_failure_is_not_treated_as_unavailable(tmp_path: Path) -> None: + """Once column bases exist, failed native compilation cannot remove coverage.""" + pair = ReleasePair("v0.5.3", "v0.5.2") + _baseline, current = prepared(tmp_path, pair) + plan = declaration(current) + assert isinstance(plan, SuiteMeasurementPlan) + suite = next(item for item in plan.suites if item.name == "column-basis") + assert suite.since == pair.baseline == "v0.5.2" + policy = suite.measurement.baseline.policy + assert isinstance(policy, DiscoveryPolicy) + assert len(policy.discovery.required) == 55 + failure = (sys.executable, "-c", 'import sys; sys.stderr.write("fixture column-basis compilation failed\\n"); sys.exit(7)') + discovery = replace(policy.discovery, suites=(replace(policy.discovery.suites[0], command=failure),)) + baseline = replace(suite.measurement.baseline, policy=replace(policy, discovery=discovery)) + selected = replace(plan, suites=(replace(suite, measurement=replace(suite.measurement, baseline=baseline)),)) + progress = io.StringIO() + with pytest.raises(subprocess.CalledProcessError) as raised: + measure_pair(current, selected, pair, working_tree=True, progress=ProgressReporter(mode="json", stream=progress)) + assert raised.value.returncode == 7 + assert b"fixture column-basis compilation failed" in raised.value.stderr + events = [json.loads(line) for line in progress.getvalue().splitlines()] + # Discovery failed before an observed inventory could establish the budget. + assert not any(event["event"] == "budget" for event in events) + assert {key: events[-1][key] for key in ("event", "state", "suite", "phase", "completed")} == { + "event": "failed", + "state": "discovery", + "suite": "column-basis", + "phase": "baseline", + "completed": 0, + } + assert not any(event["event"] == "validated" for event in events) + assert not any(event["state"] in {"gate", "acquisition"} for event in events) + + +@pytest.mark.parametrize("failed_stage", ["gate", "acquisition"]) +def test_shared_progress_keeps_early_failure_unaccepted(tmp_path: Path, failed_stage: str) -> None: + """Supported native failures remain failures, never completed progress.""" + _baseline, current = prepared(tmp_path) + plan = declaration(current) + assert isinstance(plan, SuiteMeasurementPlan) + failure = (sys.executable, "-c", "raise SystemExit(7)") + suite = plan.suites[0] + phase = replace(suite.measurement.baseline, **{("gate" if failed_stage == "gate" else "command"): failure}) + plan = replace(plan, suites=(replace(suite, measurement=replace(suite.measurement, baseline=phase)), *plan.suites[1:])) + progress = io.StringIO() + if failed_stage == "gate": + with pytest.raises(ValueError, match=r"^baseline preflight gate failed$"): + measure_pair(current, plan, PAIR, working_tree=True, progress=ProgressReporter(mode="json", stream=progress)) + else: + with pytest.raises(subprocess.CalledProcessError) as raised: + measure_pair(current, plan, PAIR, working_tree=True, progress=ProgressReporter(mode="json", stream=progress)) + assert raised.value.returncode == 7 + events = [json.loads(line) for line in progress.getvalue().splitlines()] + failed = events[-1] + assert failed["event"] == "failed" + assert failed["state"] == failed_stage + assert failed["phase"] == "baseline" + assert failed["completed"] == 0 + assert not any(event["event"] == "validated" for event in events) + assert not any(event["event"] == "finished" and event["state"] == failed_stage and event["suite"] == suite.name for event in events) + assert not (current / "target/bench-reports/full/run.json").exists() + + @pytest.fixture def complete_run(tmp_path: Path, measured_run: tuple[Path, Evidence]) -> tuple[Path, Evidence]: root, evidence = measured_run @@ -422,22 +719,244 @@ def identities(root: Path) -> list[str]: return cast("list[str]", json.loads((root / "fixture-identities.json").read_text(encoding="utf-8"))) +def test_local_candidate_capture_and_final_acceptance_reuse_original_bytes( + complete_run: tuple[Path, Evidence], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stub only the separately tested native CI provider; preserve real candidate binding.""" + root, _evidence = complete_run + current_tag, current_revision, baseline_tag, baseline_revision = identities(root) + record = serialize_preparation( + Preparation( + "acgetchell/la-stack", + "tooling/measurements/full.toml", + hashlib.sha256((root / "tooling/measurements/full.toml").read_bytes()).hexdigest(), + current_tag.removeprefix("v"), + ReleasePair(current_tag, baseline_tag), + current_revision, + baseline_revision, + working_tree=True, + ) + ) + (root / "target/bench-reports/release-inputs.json").write_bytes(record) + owner = tmp_path / "independent owner" + owner.mkdir() + freeze(owner, {"preparation.json": record}) + + def no_acquisition(*_args: object, **_kwargs: object) -> None: + pytest.fail("retained candidate consumers must never acquire or preflight") + + monkeypatch.setattr("research_repo_tools.common_measurement.run_command_live", no_acquisition) + with pytest.raises(FileNotFoundError, match=r"run\.json"): + capture(root, owner) + assertions = capture(root, owner, retained_scratch=True) + assert (owner / "run.json").read_bytes() == (root / "target/bench-reports/full/run.json").read_bytes() + assert (owner / "evidence.json").read_bytes() == (root / "target/bench-reports/full/evidence.json").read_bytes() + original = {path.name: path.read_bytes() for path in owner.iterdir()} + assert capture(root, owner) == assertions + assert {path.name: path.read_bytes() for path in owner.iterdir()} == original + publish_publication( + load_report_plan( + root, + "tooling/performance-report.toml", + payload="target/bench-reports/full/run.json", + manifest="target/bench-reports/full/evidence.json", + ) + ) + result = recipe(root, "performance-readme", *identities(root)) + assert result.returncode == 0, result.stdout + result.stderr + for args in ( + ["add", "."], + ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "final outputs"], + ): + run_command("git", args, cwd=root) + commit = run_command("git", ["--no-pager", "rev-parse", "HEAD"], cwd=root).stdout.strip() + native = deterministic_json( + { + "package": {"name": "la-stack", "version": current_tag.removeprefix("v")}, + "policy": {"repository": "acgetchell/la-stack", "guarantee": "signed-source", "event": "push", "ref": "refs/heads/main", "signing_job": "sign"}, + "run": {"head_sha": commit, "id": 123, "run_attempt": 1, "path": ".github/workflows/ci.yml"}, + "artifact": {"id": 456, "digest": "sha256:" + "a" * 64}, + } + ) + expected_selections: list[bytes | None] = [] + + def native_evidence(*_args: object, expected: bytes | None = None, **_kwargs: object) -> bytes: + expected_selections.append(expected) + if expected is not None: + assert expected == native + return native + + monkeypatch.setattr("research_repo_tools.candidate_reuse.verify_release_evidence", native_evidence) + monkeypatch.setattr("research_repo_tools.dependency_provenance.check_release_dependencies", lambda *_args, **_kwargs: None) + monkeypatch.setattr("tooling.release_candidate.check_release_dependencies", lambda *_args, **_kwargs: None) + accepted = verify(root, owner) + original_acceptance = accepted.read_bytes() + assert verify(root, owner).read_bytes() == original_acceptance + assert expected_selections == [None, native] + artifact = archive(owner, commit) + original_archive = artifact.read_bytes() + assert archive(owner, commit).read_bytes() == original_archive + assert {name: (owner / name).read_bytes() for name in original} == original + assert json.loads(original_acceptance)["binding"]["candidate"]["revision"] == current_revision + assert json.loads(original_acceptance)["binding"]["final"]["revision"] == commit + assert_hosted_reuse(root, owner, accepted, artifact, monkeypatch) + assert expected_selections == [None, native, native] + # Changed receipt bytes fail against the separately frozen digest. + (owner / "publication.json").write_bytes(original["publication.json"] + b" ") + with pytest.raises(RuntimeError, match="candidate verification failed"): + verify(root, owner) + assert accepted.read_bytes() == original_acceptance + assert artifact.read_bytes() == original_archive + + +def assert_hosted_reuse(root: Path, owner: Path, accepted: Path, artifact: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Verify the real archive and final binding through the hosted consumer.""" + tmp_path = owner.parent + original_acceptance, original_archive = accepted.read_bytes(), artifact.read_bytes() + policy = parse_candidate_policy((accepted.parent / "policy.json").read_bytes()) + current_tag, commit = policy.tag, policy.commit + assert commit is not None + native = deterministic_json(json.loads(original_acceptance)["native"]) + assertions = { + "acquisition_sha256": (owner / "acquisition.sha256").read_text(encoding="utf-8").strip(), + "publication_sha256": (owner / "publication.sha256").read_text(encoding="utf-8").strip(), + } + incoming = tmp_path / "transport with spaces" + incoming.mkdir() + files = { + "policy": accepted.parent / "policy.json", + "publication": owner / "publication.json", + "acceptance": accepted, + "archive": artifact, + } + assets = {} + for identifier, (role, source) in enumerate(files.items(), 1): + name = "archive.tar.gz" if role == "archive" else f"{role}.json" + data = source.read_bytes() + (incoming / name).write_bytes(data) + assets[role] = {"name": name, "id": identifier, "sha256": hashlib.sha256(data).hexdigest()} + transfer = json.dumps( + { + "schema": 1, + "target": json.loads(serialize_release_target(ReleaseTarget("acgetchell/la-stack", current_tag, 42, commit, current_tag))), + "acquisition": assertions["acquisition_sha256"], + "publication": assertions["publication_sha256"], + "assets": assets, + } + ).encode() + for key, value in { + "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_REPOSITORY": "acgetchell/la-stack", + "GITHUB_REF": f"refs/tags/{current_tag}", + "GITHUB_SHA": commit, + "RELEASE_TAG": current_tag, + "GITHUB_WORKFLOW_REF": f"acgetchell/la-stack/.github/workflows/release-benchmarks.yml@refs/tags/{current_tag}", + "GITHUB_ACTOR": "acgetchell", + "GITHUB_TRIGGERING_ACTOR": "acgetchell", + }.items(): + monkeypatch.setenv(key, value) + received = tmp_path / "hosted acceptance" + assert receive(root, received, transfer, source=incoming).read_bytes() == original_archive + assert (received / "acceptance.json").read_bytes() == original_acceptance + (incoming / "acceptance.json").write_bytes(original_acceptance + b" ") + with pytest.raises(ValueError, match="transported candidate bytes differ"): + receive(root, received, transfer, source=incoming) + assert (received / "acceptance.json").read_bytes() == original_acceptance + (incoming / "acceptance.json").write_bytes(original_acceptance) + + def changed_native(*_args: object, expected: bytes | None = None, **_kwargs: object) -> bytes: + assert expected == native + message = "selected run or attempt changed" + raise ValueError(message) + + with pytest.MonkeyPatch.context() as changed: + changed.setattr("research_repo_tools.candidate_reuse.verify_release_evidence", changed_native) + with pytest.raises(ValueError, match="selected run or attempt changed"): + receive(root, received, transfer, source=incoming) + assert (received / "acceptance.json").read_bytes() == original_acceptance + assert_published_renewal(root, files, transfer, monkeypatch) + + +def assert_published_renewal(root: Path, files: dict[str, Path], transfer: bytes, monkeypatch: pytest.MonkeyPatch) -> None: + """Renew real candidate acceptance while retaining every original release asset.""" + owner = files["publication"].parent + original = {role: path.read_bytes() for role, path in files.items()} + data = json.loads(transfer) + tag = data["target"]["tag"] + data["assets"]["attached"] = {"name": f"la-stack-{tag}-criterion-baseline.tar.gz", "id": 9, "sha256": data["assets"]["archive"]["sha256"]} + original["attached"] = original["archive"] + native = json.loads(original["acceptance"])["native"] + native["run"]["run_attempt"] = 2 + native["artifact"] = {"id": 789, "digest": "sha256:" + "b" * 64} + renewed_native = deterministic_json(native) + selections: list[bytes | None] = [] + + def native_evidence(*_args: object, expected: bytes | None = None, **_kwargs: object) -> bytes: + selections.append(expected) + if expected is not None: + assert expected == renewed_native + return renewed_native + + def published(*_args: object, **_kwargs: object) -> GitHubRelease: + return GitHubRelease( + repository="acgetchell/la-stack", + tag=tag, + identifier=42, + draft=False, + immutable=True, + prerelease=False, + title=tag, + assets=tuple((item["name"], item["id"], len(original[role]), item["sha256"]) for role, item in data["assets"].items()), + ) + + def download(_root: Path, _repository: str, _tag: str, name: str, destination: Path, *, expected_sha256: str, **_kwargs: object) -> None: + role = next(role for role, item in data["assets"].items() if item["name"] == name) + assert hashlib.sha256(original[role]).hexdigest() == expected_sha256 + destination.write_bytes(original[role]) + + with monkeypatch.context() as renewed: + renewed.setattr("research_repo_tools.candidate_reuse.verify_release_evidence", native_evidence) + renewed.setattr("tooling.release_candidate.check_reviewed_release", published) + renewed.setattr("tooling.release_candidate.download_release_asset", download) + renewed.setenv("GITHUB_WORKFLOW_REF", f"acgetchell/la-stack/.github/workflows/publish-crates.yml@refs/tags/{tag}") + reviewed = verify(root, owner, "renewed-ci").read_bytes() + assert reviewed != original["acceptance"] + assert json.loads(reviewed)["binding"] == json.loads(original["acceptance"])["binding"] + digest = hashlib.sha256(reviewed).hexdigest() + assertions = json.dumps(data).encode() + before, after = owner.parent / "before approval", owner.parent / "after approval" + assert receive(root, before, assertions, source=ReviewedAcceptance(digest)).read_bytes() == original["archive"] + frozen = (before / "acceptance.json").read_bytes() + assert frozen == reviewed + assert receive(root, after, assertions, source=ReviewedAcceptance(digest, frozen)).read_bytes() == original["archive"] + assert (after / "acceptance.json").read_bytes() == reviewed + assert selections == [None, None, renewed_native] + assert {role: path.read_bytes() for role, path in files.items()} == {role: original[role] for role in files} + + def test_working_tree_inputs_include_nonignored_untracked_consumer_sources(complete_run: tuple[Path, Evidence]) -> None: root, evidence = complete_run snapshot = capture_snapshot(root) files = {name: data for name, data, _mode in snapshot.untracked} assert files["src/untracked.rs"] == b"// included working-tree input\n" assert not any(name.startswith("target/") for name in files) - context = dict(dict(evidence.sources)["current"].context) - assert "src/untracked.rs" in json.loads(context["source-inventory"]) + observations = [source for _suite, phase, _sample, source in parse_suite_run(select_trial(evidence, 1)).observations if phase == "current"] + assert len(observations) == 7 + for source in observations: + assert "src/untracked.rs" in json.loads(dict(source.context)["source-inventory"]) def test_actual_non_exact_measurement_recipe_discovers_and_retains_complete_samples(tmp_path: Path) -> None: _baseline, root = prepared(tmp_path) - shutil.copytree(ROOT / "tooling/plots/non-exact", root / "tooling/plots/non-exact") + shutil.copytree(ROOT / "tooling/plots/non-exact", root / "tooling/plots/non-exact", dirs_exist_ok=True) manifest = root / "Cargo.toml" current = manifest.read_text(encoding="utf-8") - manifest.write_text(current.replace('version = "0.5.1"', 'version = "0.5.0"', 1), encoding="utf-8", newline="\n") + manifest.write_text( + current.replace(f'version = "{PAIR.current.removeprefix("v")}"', f'version = "{PAIR.baseline.removeprefix("v")}"', 1), encoding="utf-8", newline="\n" + ) + run_command("git", ["-c", "tag.gpgsign=false", "tag", "-d", PAIR.baseline], cwd=root) for arguments in ( ["add", "."], ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "baseline"], @@ -461,7 +980,9 @@ def test_actual_non_exact_measurement_recipe_discovers_and_retains_complete_samp result = recipe(root, "performance-measure", PAIR.current, PAIR.baseline, "non-exact") assert result.returncode == 0, result.stdout + result.stderr evidence = load_evidence(root / "target/bench-reports/non-exact/run.json", root / "target/bench-reports/non-exact/evidence.json") - run = validate_run_policy(evidence, declaration(root, "non-exact")) + plan = declaration(root, "non-exact") + assert isinstance(plan, CompleteMeasurementPlan) + run = validate_run_policy(evidence, plan) assert all(name.startswith("d") for _phase, sample in run.phases for name in sample.policy.expected) assert (root / "target/bench-reports/non-exact/report.md").is_file() assert not (root / "target/bench-reports/full").exists() @@ -482,11 +1003,12 @@ def test_actual_retention_offline_plot_and_readme_recipes(complete_run: tuple[Pa assert result.returncode == 0, result.stdout + result.stderr result = recipe(root, "plot-vs-linalg", *identities(root), f"lu_solve-{statistic}-{axis}", source) assert result.returncode == 0, result.stdout + result.stderr - csv = (root / f"docs/assets/bench/vs_linalg_lu_solve_{statistic}_{axis}.csv").read_text(encoding="utf-8") - assert "nalgebra" in csv - assert "faer" in csv - assert "baseline" in csv - assert statistic in csv + csv_text = (root / f"docs/assets/bench/vs_linalg_lu_solve_{statistic}_{axis}.csv").read_text(encoding="utf-8") + csv_rows = list(csv.DictReader(io.StringIO(csv_text))) + assert len(csv_rows) == 24 + assert {row["series"] for row in csv_rows} == {"lu-solve/la-stack current", "lu-solve/nalgebra", "lu-solve/faer"} + assert {row["phase"] for row in csv_rows} == {"current"} + assert {row["statistic"] for row in csv_rows} == {statistic} if source == "retained": prepared_inputs = root / "target/bench-reports/release-inputs.json" prepared_inputs.parent.mkdir(parents=True) @@ -512,7 +1034,7 @@ def test_actual_retention_offline_plot_and_readme_recipes(complete_run: tuple[Pa assert "Speedup vs faer" in document assert "Complete run evidence" in document assert "Evidence envelope" in document - assert "la-stack v0.5.1" in document + assert "la-stack v0.5.2" in document assert "Statistic: median. Unit: ns." in document catalog = tomllib.loads((ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) rows = [ @@ -524,7 +1046,7 @@ def test_actual_retention_offline_plot_and_readme_recipes(complete_run: tuple[Pa assert len(row) == 6 expected = [ timings[phase][f"d{dimension}/{benchmark}"]["median"] - for phase, benchmark in zip(("current", "baseline", "baseline"), catalog["metrics"]["lu_solve"]["benchmarks"], strict=True) + for phase, benchmark in zip(("current", "current", "current"), catalog["metrics"]["lu_solve"]["benchmarks"], strict=True) ] assert [float(cell.split("
")[0]) for cell in row[1:4]] == pytest.approx(expected) # Published ratios have six significant digits; the oracle uses the @@ -533,11 +1055,15 @@ def test_actual_retention_offline_plot_and_readme_recipes(complete_run: tuple[Pa [expected[1] / expected[0], expected[2] / expected[0]], rel=1e-5 ) checked = recipe(root, "markdown-check") - assert checked.returncode == 0, checked.stderr + assert checked.returncode == 0, checked.stdout + checked.stderr @pytest.mark.parametrize("failure", ["partial-scratch", "stale-source", "stale-harness", "formatter"]) -def test_publication_boundaries_leave_all_outputs_unchanged(complete_run: tuple[Path, Evidence], failure: str) -> None: +def test_publication_boundaries_leave_all_outputs_unchanged( + complete_run: tuple[Path, Evidence], + failure: str, + capsys: pytest.CaptureFixture[str], +) -> None: root, _evidence = complete_run result = recipe(root, "performance-promote") assert result.returncode == 0, result.stdout + result.stderr @@ -547,22 +1073,46 @@ def test_publication_boundaries_leave_all_outputs_unchanged(complete_run: tuple[ (root / "src/lib.rs").write_text("// changed\n", encoding="utf-8", newline="\n") elif failure == "stale-harness": (root / "tooling/performance.just").write_text("# changed\n", encoding="utf-8", newline="\n") - else: - config = root / "tooling/performance-readme.toml" - config.write_text( - config.read_text(encoding="utf-8").replace('formatter = "pyproject.toml"', 'formatter = "bad-formatter.toml"'), encoding="utf-8", newline="\n" - ) - (root / "bad-formatter.toml").write_text("[invalid TOML\n", encoding="utf-8", newline="\n") outputs = [root / "README.md", *(root / "docs").rglob("*")] before = {path: path.read_bytes() for path in outputs if path.is_file()} - result = recipe(root, "performance-promote") if failure == "partial-scratch" else recipe(root, "performance-readme", *identities(root)) + if failure == "formatter": + # Frozen publisher configuration must not change after measurement. + # Simulate a transient formatter process failure at its real boundary. + def failed_formatter(*_args: object, **_kwargs: object) -> None: + raise subprocess.CalledProcessError(7, ["rumdl"], stderr="fixture formatter failure") + + with pytest.MonkeyPatch.context() as patch: + patch.setattr("research_repo_tools.postprocess_changelog.run_safe_command", failed_formatter) + current, revision, baseline, previous = identities(root) + status = main( + [ + "--root", + str(root), + "performance", + "readme", + "tooling/performance-release.toml", + "--current-release", + current, + "--current-revision", + revision, + "--baseline-release", + baseline, + "--baseline-revision", + previous, + ] + ) + diagnostic = capsys.readouterr() + result = subprocess.CompletedProcess([], status, diagnostic.out, diagnostic.err) + else: + result = recipe(root, "performance-promote") if failure == "partial-scratch" else recipe(root, "performance-readme", *identities(root)) assert result.returncode != 0 - if failure == "stale-source": - assert "current-sources differ" in result.stderr - elif failure == "stale-harness": - assert "current-harness differ" in result.stderr - elif failure == "formatter": - assert "rumdl could not format README.md" in result.stderr + diagnostics = { + "stale-source": "current core-default source-inventory differs from measured inputs", + "stale-harness": "current core-default harness-inventory differs from measured inputs", + "formatter": "rumdl could not format README.md", + } + if failure in diagnostics: + assert diagnostics[failure] in result.stderr if failure == "partial-scratch": result = recipe(root, "plot-vs-linalg", *identities(root), "lu_solve-median-log", "scratch") assert result.returncode != 0 @@ -572,8 +1122,6 @@ def test_publication_boundaries_leave_all_outputs_unchanged(complete_run: tuple[ # Retry publication after repairing its formatter. Archived measurements # remain byte-identical; no timing or discovery command is involved. retained = {path: path.read_bytes() for path in (root / "docs/performance-runs").rglob("*") if path.is_file()} - config = root / "tooling/performance-readme.toml" - config.write_bytes((ROOT / "tooling/performance-readme.toml").read_bytes()) result = recipe(root, "performance-readme", *identities(root)) assert result.returncode == 0, result.stdout + result.stderr assert {path: path.read_bytes() for path in retained} == retained @@ -592,7 +1140,9 @@ def test_actual_measurement_recipe_rejects_old_releases_before_git(phase: str, t shutil.copyfile(source, destination) (root / "src").mkdir() (root / "src/lib.rs").write_text("// fixture\n", encoding="utf-8", newline="\n") - result = recipe(root, "performance-measure", "v0.4.6" if phase == "current" else PAIR.current, "v0.4.6" if phase == "baseline" else PAIR.baseline) + result = recipe( + root, "performance-measure", "v0.4.6" if phase == "current" else PAIR.current, "v0.4.6" if phase == "baseline" else PAIR.baseline, "non-exact" + ) assert result.returncode != 0 assert "requires v0.5.0 or newer" in result.stderr assert not (root / ".git").exists() @@ -618,7 +1168,7 @@ def test_pre_declaration_complete_run_is_preserved_as_a_frozen_record() -> None: "run.json": "9b860d8a56a4313d60031039fd518f142c1aa2e74933fc9e73fb128a00023b2a", } evidence = load_evidence(directory / "run.json", directory / "evidence.json") - with pytest.raises(ValueError, match="acquisition declaration"): + with pytest.raises(ValueError, match=r"scheduled run fields must be \['declaration', 'schema', 'trials'\]"): validate_run_policy(evidence, declaration(ROOT), release=True) assert {name: hashlib.sha256((directory / name).read_bytes()).hexdigest() for name in hashes} == hashes diff --git a/tests/tooling/test_release_baseline.py b/tests/tooling/test_release_baseline.py index 6fbd472..5c94229 100644 --- a/tests/tooling/test_release_baseline.py +++ b/tests/tooling/test_release_baseline.py @@ -8,23 +8,20 @@ import subprocess import sys import tarfile -import textwrap import tomllib from pathlib import Path from urllib.parse import urlsplit import pytest -from packaging.requirements import Requirement +import yaml from research_repo_tools.criterion_inventory import CriterionInventory, load_discovery, serialize_inventory from research_repo_tools.host_metadata import HostMetadata, serialize_host from research_repo_tools.process import run_command +from tests.tooling.test_toolchain_integration import TOOLING_REQUIREMENT + REPO_ROOT = Path(__file__).resolve().parents[2] -SHARED_REQUIREMENT = Requirement(tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["dependency-groups"]["tooling"][0]) -# Release orchestration needs the core package independently of contributor extras. -TOOLING_REQUIREMENT = SHARED_REQUIREMENT.name + str(SHARED_REQUIREMENT.specifier) WORKFLOW = REPO_ROOT / ".github/workflows/release-benchmarks.yml" -PREPARATION = REPO_ROOT / ".github/actions/prepare-release-benchmarks/action.yml" CATALOG = tomllib.loads((REPO_ROOT / "tooling/performance-policy.toml").read_text(encoding="utf-8")) REQUIRED = set(CATALOG["release"]["default"] + CATALOG["release"]["exact"]) # Retain optional diagnostic additions without letting a declaration omission @@ -84,19 +81,6 @@ def verify_saved(root: Path, *, suite: str | None = None, criterion_dir: str = " ) -def shell_step(name: str, source: Path = WORKFLOW) -> str: - """Extract a literal run block so tests exercise the workflow's actual shell.""" - text = source.read_text(encoding="utf-8") - indent = " " if source == PREPARATION else " " - step = text.split(f"{indent}- name: {name}\n", 1)[1].split(f"\n{indent}- ", 1)[0] - lines: list[str] = [] - for line in step.split(f"{indent} run: |\n", 1)[1].splitlines(): - if line.strip() and not line.startswith(f"{indent} "): - break - lines.append(line) - return textwrap.dedent("\n".join(lines)).rstrip() - - def run_shell(script: str, cwd: Path, extra_env: dict[str, str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]: bash = shutil.which("bash") assert bash is not None @@ -135,22 +119,62 @@ def package_suites(root: Path, manifest: Path, criterion: Path) -> None: for benchmark in ids: storage = STORAGE[benchmark] shutil.copytree(criterion / storage, output / storage) - # Substitute only native version/hash observations. The real uvx command - # installs the exact published suite package independently of root tooling. - probes = ( - f'rustc() {{ printf "rustc fixture\\nLLVM fixture\\n"; }}\ncargo() {{ echo "cargo fixture"; }}\nsha256sum() {{ echo "{"f" * 64} Cargo.lock"; }}\n' - ) - result = run_shell( - probes + shell_step("Package Criterion suite"), - shard, - {**SUITE_ENV, "BENCH_SUITE": suite, "RUNNER_NAME": suite, "ImageOS": "ubuntu24", "ImageVersion": "fixture-image"}, - timeout=180, + # Retain compatibility with previously published suite archives using + # the public installed packer; new releases use candidate archives. + result = run_command( + "uv", + [ + "run", + "--locked", + "--no-sync", + "research-repo-tools", + "--root", + str(shard), + "performance", + "pack-suite", + "tooling/criterion-saved.toml", + suite, + "--saved-sample", + BASELINE, + "--output", + f"target/release-suites/{suite}/criterion.tar.gz", + "--revision", + "a" * 40, + "--session", + "owner/repo:123:2", + "--host", + "target/release-host.json", + "--started-at", + "100", + "--finished-at", + "200", + "--context", + f"command=just bench-save-baseline {BASELINE} {suite}", + "--context", + "features=" + ("bench,exact" if suite == "public-exact" else "bench"), + "--context", + f"runner={suite}", + "--context", + "image=ubuntu24", + "--context", + "image_version=fixture-image", + "--context", + "rustc=rustc fixture;LLVM fixture", + "--context", + "cargo=cargo fixture", + "--context", + "cargo_lock_sha256=" + "f" * 64, + ], + cwd=REPO_ROOT, + check=False, ) assert result.returncode == 0, result.stdout + result.stderr shutil.move(shard / "target/release-suites" / suite, destination / suite) + if suite == "public-default": + shutil.copyfile(manifest, destination / suite / "inventory.json") -def test_complete_dataset_round_trips_through_workflow_archive(dataset: tuple[Path, Path], tmp_path: Path) -> None: +def test_complete_dataset_round_trips_through_legacy_suite_archive(dataset: tuple[Path, Path], tmp_path: Path) -> None: criterion, manifest = dataset criterion = criterion.rename(tmp_path / "original-criterion") package_suites(tmp_path, manifest, criterion) @@ -163,7 +187,14 @@ def test_complete_dataset_round_trips_through_workflow_archive(dataset: tuple[Pa manifest.unlink() output = tmp_path / "step outputs" asset = f"la-stack-{BASELINE}-criterion-baseline.tar.gz" - script = shell_step("Assemble release Criterion baseline") + script = ( + "set -euo pipefail\n" + "cp target/release-suites/public-default/inventory.json target/release-benchmark-inventory.json\n" + "uv run --locked --no-sync research-repo-tools performance assemble-suites tooling/criterion-saved.toml " + "target/release-suites/public-default/criterion.tar.gz target/release-suites/public-exact/criterion.tar.gz " + '--saved-sample "$RELEASE_TAG" --output "la-stack-$RELEASE_TAG-criterion-baseline.tar.gz" ' + '--revision "$GITHUB_SHA" --session "$GITHUB_REPOSITORY:$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT"\n' + ) env = {**SUITE_ENV, "GITHUB_WORKSPACE": tmp_path.as_posix(), "GITHUB_OUTPUT": output.as_posix()} for assertion in ({"GITHUB_SHA": "b" * 40}, {"GITHUB_RUN_ATTEMPT": "3"}): rejected = run_shell(script, tmp_path, {**env, **assertion}, timeout=180) @@ -174,7 +205,7 @@ def test_complete_dataset_round_trips_through_workflow_archive(dataset: tuple[Pa # Git Bash accepts forward-slash drive paths on Windows as well as POSIX paths. result = run_shell(script, tmp_path, env, timeout=180) assert result.returncode == 0, result.stdout + result.stderr - assert output.read_text(encoding="utf-8") == f"asset={asset}\nartifact-name=bench-baseline-{BASELINE}-123-2\n" + assert not output.exists() assert_archive_contents(tmp_path / asset, criterion) # Release readers keep the standard tooling environment. Exercise its # documented extraction and full-dataset verification against the new archive. @@ -311,172 +342,51 @@ def test_scientific_inventory_detects_a_nonsentinel_omission(tmp_path: Path, dec assert_scientific_inventory(tmp_path) -def test_workflow_requires_preflight_and_isolates_publication_permissions() -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - preparation = PREPARATION.read_text(encoding="utf-8") - assert preparation.index("- name: Inventory full release suites") < preparation.index("- name: Validate benchmark inputs") - assert "run: just test-bench-inputs" in preparation - assert "run: just bench-release-inventory" in preparation - assert "continue-on-error" not in preparation - assert preparation.index("- name: Require fresh release preflight") < preparation.index("- name: Set up declared tools without caches") - assert "uses: ./.github/actions/setup-tools" in preparation - assert "cache: false" in preparation - steps = [ - "Prepare release benchmarks", - "Capture runner host", - "Save Criterion suite", - "Package Criterion suite", - "Upload temporary suite artifact", - "Assemble release Criterion baseline", - "Upload temporary baseline artifact", - ] - positions = [workflow.index(f"- name: {step}") for step in steps] - assert positions == sorted(positions) - for step in steps: - body = workflow.split(f" - name: {step}\n", 1)[1].split("\n - ", 1)[0] - assert "\n if:" not in body - assert "continue-on-error" not in workflow - assert "workflow_dispatch:" in workflow - assert "required: true" in workflow - assert "\n release:" not in workflow - assert "group: release-benchmarks-${{ inputs.tag }}" in workflow - assert "cancel-in-progress: false" in workflow - preflight, remainder = workflow.split(" measure-baseline:\n", 1) - producer, remainder = remainder.split(" release-baseline:\n", 1) - _assembler, publisher = remainder.split(" publish-baseline:\n", 1) - assert "needs: validate-release" in producer - assert "validated-attempt: ${{ github.run_attempt }}" in preflight - assert "validated-attempt: ${{ needs.validate-release.outputs.validated-attempt }}" in producer - assert "ref: ${{ github.sha }}" in producer - assert "ref: ${{ needs.validate-release.outputs.commit }}" not in producer - assert "contents: write" not in producer - assert "persist-credentials: false" in producer - assert "contents: write" in preflight - assert "actions/checkout@" not in preflight + publisher - assert "target: ${{ steps.target.outputs.target }}" in preflight - assert "RELEASE_TARGET: ${{ needs.validate-release.outputs.target }}" in publisher - assert '--expected-title "$RELEASE_TAG"' in preflight - assert '--target "$RUNNER_TEMP/release-target.json"' in publisher - assert "--publish" not in publisher - assert (preflight + publisher).count(f"--from {TOOLING_REQUIREMENT}") == 2 - assert "uses: ./" not in preflight + publisher - assert "--clobber" not in workflow - assert "needs: [validate-release, release-baseline]" in publisher - assert "GH_REPO: ${{ github.repository }}" in publisher - assert "contents: write" in publisher - assert "name: ${{ needs.release-baseline.outputs.artifact-name }}" in publisher - assert "performance host --output target/release-host.json" in producer - assert "--host target/release-host.json" in producer - assert "path: target/release-suites/${{ matrix.suite }}" in producer - assert "tar -" not in workflow - assert "tooling.release_benchmarks" not in (REPO_ROOT / "justfile").read_text(encoding="utf-8") - - -def test_parallel_suites_require_both_current_attempts_before_assembly() -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - producer, remainder = workflow.split(" measure-baseline:\n", 1)[1].split(" release-baseline:\n", 1) - assembler = remainder.split(" publish-baseline:\n", 1)[0] - assert "fail-fast: false" in producer - assert "suite: [public-default, public-exact]" in producer - assert "max-parallel: 1" not in producer - assert "BENCH_SUITE: ${{ matrix.suite }}" in producer - assert 'just bench-save-baseline "$RELEASE_TAG" "$BENCH_SUITE"' in producer - assert "performance pack-suite" in producer - assert 'tooling/criterion-saved.toml "$BENCH_SUITE" --saved-sample "$RELEASE_TAG"' in producer - assert "ref: ${{ github.sha }}" in assembler - assert "persist-credentials: false" in assembler - assert "contents: write" not in assembler - assert "needs: [validate-release, measure-baseline]" in assembler - assert "always()" not in assembler - for suite in IDS: - assert f"name: bench-suite-${{{{ inputs.tag }}}}-${{{{ github.run_id }}}}-${{{{ github.run_attempt }}}}-{suite}" in assembler - assert f"path: target/release-suites/{suite}" in assembler - assert f"target/release-suites/{suite}/criterion.tar.gz" in assembler - assert "performance assemble-suites tooling/criterion-saved.toml" in assembler - for job in (producer, assembler): - assert '--session "$GITHUB_REPOSITORY:$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT"' in job - assert '--revision "$GITHUB_SHA"' in job - for job in (producer, assembler): - assert f"--from {TOOLING_REQUIREMENT}" in job - - -def test_setup_limits_preserve_benchmark_and_tail_budgets() -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - producer = workflow.split(" measure-baseline:\n", 1)[1].split(" release-baseline:\n", 1)[0] - job_limit = re.search(r"(?m)^ timeout-minutes: (\d+)$", producer) - assert job_limit is not None - setup = producer.split(" steps:\n", 1)[1].split(" - name: Save Criterion suite", 1)[0] - setup_steps = re.findall(r"(?ms)^ - .*?(?=^ - |\Z)", setup) - assert setup_steps - setup_limits: list[int] = [] - for step in setup_steps: - limit = re.search(r"(?m)^ timeout-minutes: (\d+)$", step) - assert limit is not None, f"unbounded setup step: {step}" - setup_limits.append(int(limit[1])) - assert all(limit > 0 for limit in setup_limits) - discovery = load_discovery(REPO_ROOT, "tooling/criterion-inventory.toml") - discovery_minutes = sum(suite.timeout for suite in discovery.suites) / 60 - # The failed hosted run spent 27 minutes on uncached tool installation. - # Reserve 35 for tools, each declared cold discovery deadline, and 5 for inputs. - preparation = re.search(r"(?ms)^ - name: Prepare release benchmarks\n.*?^ timeout-minutes: (\d+)$", setup) - assert preparation is not None - assert int(preparation[1]) >= 35 + discovery_minutes + 5 - assert "uses: ./.github/actions/prepare-release-benchmarks" in setup - assert "using: composite" in PREPARATION.read_text(encoding="utf-8") - benchmark_limit = re.search(r"(?ms)^ - name: Save Criterion suite\n.*?^ timeout-minutes: (\d+)$", producer) - assert benchmark_limit is not None - assert int(benchmark_limit[1]) == 240 - for suite in ("default", "exact"): - # Keep at least 50% headroom beyond the documented planning rate. - assert int(benchmark_limit[1]) * 60 >= len(CATALOG["release"][suite]) * 12 * 1.5 - assert int(job_limit[1]) - sum(setup_limits) - int(benchmark_limit[1]) >= 20 - assert int(job_limit[1]) <= 360 - - -@pytest.mark.parametrize("suite", ["public-default", "public-exact"]) -def test_failed_suite_retains_timing_and_does_not_mark_completion(tmp_path: Path, suite: str) -> None: - (tmp_path / "target").mkdir() - # A shell function substitutes only the benchmark command; timestamps and - # failure handling execute exactly as checked into the workflow. - script = "just() { return 7; }\n" + shell_step("Save Criterion suite") - result = run_shell(script, tmp_path, {"RELEASE_TAG": BASELINE, "BENCH_SUITE": suite}) - assert result.returncode == 7 - assert (tmp_path / f"target/{suite}-started").is_file() - assert not (tmp_path / f"target/{suite}-finished").exists() - summary = tmp_path / "suite summary.md" - result = run_shell( - shell_step("Suite timing summary"), - tmp_path, - { - "GITHUB_STEP_SUMMARY": summary.as_posix(), - "BENCH_SUITE": suite, - "SUITE_OUTCOME": "failure", - }, - ) - assert result.returncode == 0, result.stderr - assert f"| {suite} | failure |" in summary.read_text(encoding="utf-8") - assert "| 240 |" in summary.read_text(encoding="utf-8") +def test_workflow_reuses_original_acquisition_and_isolates_write_credentials() -> None: + workflow = yaml.safe_load(WORKFLOW.read_bytes()) + assert set(workflow[True]) == {"workflow_dispatch"} + assert set(workflow[True]["workflow_dispatch"]["inputs"]) == {"tag", "candidate"} + assert workflow["concurrency"]["cancel-in-progress"] is False + jobs = workflow["jobs"] + assert set(jobs) == {"fetch", "verify", "attach"} + assert jobs["verify"]["needs"] == "fetch" + assert jobs["attach"]["needs"] == "verify" + assert jobs["verify"]["permissions"] == {"contents": "read", "checks": "read", "actions": "read"} + assert "github.triggering_actor == 'acgetchell'" in jobs["fetch"]["if"] + for name in ("fetch", "attach"): + job = jobs[name] + assert job["permissions"]["contents"] == "write" + assert not any("checkout@" in step.get("uses", "") or step.get("uses", "").startswith("./") for step in job["steps"]) + assert job["steps"][0]["with"]["enable-cache"] is False + assert job["steps"][0]["with"]["cache-python"] is False + verifier = jobs["verify"]["steps"] + assert verifier[0]["with"] == {"ref": "${{ github.sha }}", "fetch-depth": 0, "persist-credentials": False} + assert verifier[1]["with"]["cache"] is False + assert verifier[3]["with"]["artifact-ids"] == "${{ needs.fetch.outputs.artifact-id }}" + assert verifier[4]["env"]["CANDIDATE_TRANSFER"] == "${{ inputs.candidate }}" + assert 'release-receive "$RUNNER_TEMP/accepted-candidate" --incoming "$RUNNER_TEMP/incoming-candidate"' in verifier[4]["run"] + attach = jobs["attach"]["steps"] + assert attach[2]["with"]["artifact-ids"] == "${{ needs.verify.outputs.artifact-id }}" + assert attach[1]["env"]["ARTIFACT_DIGEST"] == "${{ needs.verify.outputs.artifact-digest }}" + text = WORKFLOW.read_text(encoding="utf-8") + assert text.count(f"--from {shlex.quote(TOOLING_REQUIREMENT)}") == 2 + for forbidden in ("bench-save-baseline", "measure-baseline", "cargo bench", "performance measure", "--clobber", "continue-on-error"): + assert forbidden not in text + assert "publish_release_asset(root, target.repository, target.tag, artifact, target=target)" in text + assert "publish=True" not in text @pytest.mark.parametrize("validated_attempt", ["", "1", "2"]) -def test_benchmark_retry_requires_current_preflight(tmp_path: Path, validated_attempt: str) -> None: +def test_retry_requires_current_retrieval(tmp_path: Path, validated_attempt: str) -> None: + workflow = yaml.safe_load(WORKFLOW.read_bytes()) result = run_shell( - shell_step("Require fresh release preflight", PREPARATION), + workflow["jobs"]["verify"]["steps"][2]["run"], tmp_path, {"VALIDATED_ATTEMPT": validated_attempt, "GITHUB_RUN_ATTEMPT": "2"}, ) assert result.returncode == (0 if validated_attempt == "2" else 1) -@pytest.mark.parametrize("ref", ["refs/heads/main", f"refs/heads/{BASELINE}", "refs/tags/v9.9.9", ""]) -def test_preflight_rejects_dispatch_outside_release_tag(tmp_path: Path, ref: str) -> None: - output = tmp_path / "outputs" - result = run_shell(shell_step("Validate draft release target"), tmp_path, {"RELEASE_TAG": BASELINE, "GITHUB_REF": ref, "GITHUB_OUTPUT": output.as_posix()}) - assert result.returncode == 1 - assert f"Dispatch with --ref {BASELINE}" in result.stdout - assert not output.exists() - - GH_FIXTURE = r"""import hashlib, json, os, sys from pathlib import Path from urllib.parse import urlsplit @@ -545,19 +455,32 @@ def test_installed_cli_binds_trusted_target_and_leaves_draft_for_review(tmp_path # Substitute only uvx's environment acquisition; retain its exact package # arguments and execute the already locked package in a real child process. runner = installed_runner() - result = run_shell(runner + shell_step("Validate draft release target"), fixture, env) + preflight = ( + "set -euo pipefail\n" + f"uvx --python 3.15 --from {shlex.quote(TOOLING_REQUIREMENT)} research-repo-tools " + '--root "$RUNNER_TEMP" performance release-draft "$GH_REPO" "$RELEASE_TAG" ' + '--commit "$GITHUB_SHA" --expected-title "$RELEASE_TAG" > "$GITHUB_OUTPUT"\n' + ) + upload = ( + "set -euo pipefail\n" + 'printf "%s" "$RELEASE_TARGET" > "$RUNNER_TEMP/release-target.json"\n' + f"uvx --python 3.15 --from {shlex.quote(TOOLING_REQUIREMENT)} research-repo-tools " + '--root "$RUNNER_TEMP" performance release-upload "$GH_REPO" "$RELEASE_TAG" ' + '"$RUNNER_TEMP/$RELEASE_ASSET" --target "$RUNNER_TEMP/release-target.json"\n' + ) + result = run_shell(runner + preflight, fixture, env) assert result.returncode == 0, result.stderr target = output.read_text(encoding="utf-8").strip().removeprefix("target=") assert json.loads(target)["release_id"] == 42 assert json.loads(target)["commit"] == "a" * 40 assert json.loads(target)["expected_title"] == BASELINE - stale = run_shell(runner + shell_step("Attach baseline for review"), fixture, {**env, "RELEASE_TARGET": target, "REMOTE_COMMIT": "b" * 40}) + stale = run_shell(runner + upload, fixture, {**env, "RELEASE_TARGET": target, "REMOTE_COMMIT": "b" * 40}) assert stale.returncode != 0 assert not (fixture / "uploaded").exists() assert not (fixture / "published").exists() stale_calls = [json.loads(line) for line in (fixture / "api-calls").read_text(encoding="utf-8").splitlines()] assert not any(urlsplit(item).hostname == "uploads.github.com" for call in stale_calls for item in call) - result = run_shell(runner + shell_step("Attach baseline for review"), fixture, {**env, "RELEASE_TARGET": target}) + result = run_shell(runner + upload, fixture, {**env, "RELEASE_TARGET": target}) assert result.returncode == 0, result.stderr assert (fixture / "uploaded").exists() assert not (fixture / "published").exists() diff --git a/tests/tooling/test_release_performance.py b/tests/tooling/test_release_performance.py index 6900bca..a0dfa87 100644 --- a/tests/tooling/test_release_performance.py +++ b/tests/tooling/test_release_performance.py @@ -12,9 +12,12 @@ from research_repo_tools import performance_release from research_repo_tools.cli import main from research_repo_tools.just_inspect import dry_run +from research_repo_tools.performance_progress import ProgressReporter from research_repo_tools.process import run_command, run_git_command from research_repo_tools.releases import PublishedRelease +from tooling import release_candidate as candidate + ROOT = Path(__file__).resolve().parents[2] INPUTS = "target/bench-reports/release-inputs.json" @@ -32,7 +35,10 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "absent-gitconfig")) monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") run_git_command(["--no-pager", "init", "--quiet"], cwd=root) - for tag in ("v0.5.0", "v0.5.1", "head"): + for tag in ("v0.4.6", "v0.5.0", "v0.5.1", "head"): + release = "0.5.2" if tag == "head" else tag.removeprefix("v") + (root / "Cargo.toml").write_text(f'[package]\nname = "fixture"\nversion = "{release}"\n', encoding="utf-8", newline="\n") + run_git_command(["--no-pager", "add", "Cargo.toml"], cwd=root) run_git_command( [ "--no-pager", @@ -163,15 +169,64 @@ def test_missing_or_legacy_preparation_stops_before_readme_and_measurement(consu assert not (consumer / "target/bench-reports/full").exists() +@pytest.mark.parametrize("owner_kind", ["directory", "file"]) +def test_fresh_acquisition_preserves_existing_owner_with_recovery_guidance(consumer: Path, monkeypatch: pytest.MonkeyPatch, owner_kind: str) -> None: + releases(monkeypatch, "v0.5.1") + performance_release.prepare_performance(consumer, settings(consumer)) + owner = consumer.parent / "independent owner" + if owner_kind == "directory": + owner.mkdir() + retained = owner / "preparation.json" if owner_kind == "directory" else owner + original = b"original owner bytes\r\n\xff" + retained.write_bytes(original) + monkeypatch.setattr(candidate, "check_release_dependencies", lambda _root: None) + monkeypatch.setattr(candidate, "freeze", lambda *_args, **_kwargs: pytest.fail("existing owner must be rejected before freezing")) + monkeypatch.setattr(candidate, "measure_release", lambda *_args, **_kwargs: pytest.fail("existing owner must not start acquisition")) + + with pytest.raises(ValueError, match="acquisition owner record already exists") as stopped: + candidate.acquire(consumer, owner, "target/checkpoint", resume=False, progress=ProgressReporter(mode="none")) + + assert str(owner) in str(stopped.value) + assert "new checkpoint/owner directory" in str(stopped.value) + assert "performance-resume if the original checkpoint still exists" in str(stopped.value) + assert retained.read_bytes() == original + if owner_kind == "directory": + assert list(owner.iterdir()) == [retained] + assert not (consumer / "target/checkpoint").exists() + + +def test_fresh_acquisition_preserves_unrelated_directory_creation_error(consumer: Path, monkeypatch: pytest.MonkeyPatch) -> None: + releases(monkeypatch, "v0.5.1") + performance_release.prepare_performance(consumer, settings(consumer)) + owner = consumer.parent / "independent owner" + failure = FileExistsError("unrelated directory creation failure") + + def fail_mkdir(_self: Path, *_args: object, **_kwargs: object) -> None: + raise failure + + monkeypatch.setattr(candidate, "check_release_dependencies", lambda _root: None) + monkeypatch.setattr(Path, "mkdir", fail_mkdir) + with pytest.raises(FileExistsError) as stopped: + candidate.acquire(consumer, owner, "target/checkpoint", resume=False, progress=ProgressReporter(mode="none")) + + assert stopped.value is failure + assert not owner.exists() + + def test_release_prepares_before_measurement_and_readme_remains_separate() -> None: preparation = dry_run(ROOT, "performance-prepare").stderr assert 'performance prepare tooling/performance-release.toml "$@"' in preparation assert "git fetch" not in preparation measurement = dry_run(ROOT, "performance-release").stderr assert measurement.count("performance prepare tooling/performance-release.toml --fetch-tags --allow-git-mutations") == 1 - assert measurement.count("performance release tooling/performance-release.toml") == 1 - assert measurement.index("performance prepare") < measurement.index("performance release") + assert measurement.count("--release-intent") == 1 + assert measurement.count("tooling/release_candidate.py acquire") == 1 + assert measurement.index("performance prepare") < measurement.index("tooling/release_candidate.py acquire") assert "performance readme" not in measurement + assert "--progress 'human' --progress-cadence '300'" in measurement + captured = dry_run(ROOT, "performance-release", ["json", "60"]).stderr + assert "--progress 'json' --progress-cadence '60'" in captured + assert captured.count("tooling/release_candidate.py acquire") == 1 publication = dry_run(ROOT, "performance-readme").stderr assert "performance readme tooling/performance-release.toml" in publication assert "performance measure" not in publication @@ -179,8 +234,9 @@ def test_release_prepares_before_measurement_and_readme_remains_separate() -> No assert "performance release" not in publication +@pytest.mark.parametrize("mode", ["fresh", "resume"]) @pytest.mark.parametrize("failed_phase", ["none", "prepare", "release"]) -def test_release_recipe_stops_on_failure_without_publishing_readme(tmp_path: Path, failed_phase: str) -> None: +def test_release_recipe_stops_on_failure_without_publishing_readme(tmp_path: Path, failed_phase: str, mode: str) -> None: """Execute the real Bash recipe with a bounded CLI substitute, never fetching or timing.""" root = tmp_path / "release with spaces" root.mkdir() @@ -191,7 +247,7 @@ def test_release_recipe_stops_on_failure_without_publishing_readme(tmp_path: Pat "#!/usr/bin/env bash\n" "set -euo pipefail\n" 'printf "%s\\n" "$*" >> "$PERFORMANCE_COMMAND_LOG"\n' - 'if [[ " $* " == *" performance $PERFORMANCE_FAIL_PHASE "* ]]; then\n' + 'if [[ " $* " == *" performance $PERFORMANCE_FAIL_PHASE "* || ( "$PERFORMANCE_FAIL_PHASE" == release && " $* " == *" acquire "* ) ]]; then\n' ' printf "fixture %s failure\\n" "$PERFORMANCE_FAIL_PHASE" >&2\n' " exit 7\n" "fi\n", @@ -202,7 +258,7 @@ def test_release_recipe_stops_on_failure_without_publishing_readme(tmp_path: Pat log = root / "commands.log" result = run_command( "just", - ["--no-deps", "--justfile", str(ROOT / "justfile"), "--working-directory", str(root), "performance-release"], + ["--no-deps", "--justfile", str(ROOT / "justfile"), "--working-directory", str(root), "performance-release", "human", "300", "target/checkpoint", mode], cwd=root, env=os.environ | { @@ -213,12 +269,26 @@ def test_release_recipe_stops_on_failure_without_publishing_readme(tmp_path: Pat check=False, ) commands = [shlex.split(line) for line in log.read_text(encoding="utf-8").splitlines()] - phases = [command[command.index("performance") + 1 :] for command in commands] - expected = [["prepare", "tooling/performance-release.toml", "--fetch-tags", "--allow-git-mutations"]] - if failed_phase != "prepare": - expected.append(["release", "tooling/performance-release.toml", "--allow-git-mutations"]) + phases = [ + command[command.index("performance") + 1 :] if "performance" in command else command[command.index("tooling/release_candidate.py") + 1 :] + for command in commands + ] + expected = [["prepare", "tooling/performance-release.toml", "--fetch-tags", "--allow-git-mutations", "--release-intent"]] if mode == "fresh" else [] + if failed_phase != "prepare" or mode == "resume": + expected.append( + [ + "acquire", + "--progress", + "human", + "--progress-cadence", + "300", + "--checkpoint", + "target/checkpoint", + ] + + (["--resume"] if mode == "resume" else []) + ) assert phases == expected - status = 0 if failed_phase == "none" else 7 + status = 0 if failed_phase == "none" or (mode == "resume" and failed_phase == "prepare") else 7 assert result.returncode == status, result.stderr if status: assert f"fixture {failed_phase} failure" in result.stderr diff --git a/tests/tooling/test_release_publishing.py b/tests/tooling/test_release_publishing.py index 99c704d..aa4074c 100644 --- a/tests/tooling/test_release_publishing.py +++ b/tests/tooling/test_release_publishing.py @@ -14,7 +14,7 @@ import pytest import yaml -from research_repo_tools import changelog, config, process, release_publishing +from research_repo_tools import changelog, config, dependency_provenance, process, release_publishing from research_repo_tools.cli import main from research_repo_tools.just_inspect import dry_run from research_repo_tools.process import run_command @@ -31,6 +31,7 @@ def test_publishing_policy_requires_native_scientific_gates_and_the_criterion_asset() -> None: + assert POLICY.release.published_dependencies == ("research-repo-tools",) assert POLICY.publishing is not None assert (POLICY.publishing.registry, POLICY.publishing.package, POLICY.publishing.repository) == ("crates-io", "la-stack", "acgetchell/la-stack") assert tuple((check.name, check.app_id) for check in POLICY.publishing.required_checks) == tuple((name, 15368) for name in CHECKS) @@ -71,6 +72,15 @@ def git(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: f"commits/{SHA}/check-runs?filter=latest&per_page=100": [{"total_count": len(checks), "check_runs": checks}], } monkeypatch.setattr(release_publishing, "run_git_command", git) + + # These fixtures model a published supplier while testing native release + # identities. Separate consumer tests exercise the real development-pin gate. + def published(root: Path, *, packages: tuple[str, ...] | None = None) -> tuple[dependency_provenance.DependencyIdentity, ...]: + assert root == REPO_ROOT + assert packages == ("research-repo-tools",) + return (dependency_provenance.DependencyIdentity("research-repo-tools", "0.2.6", "https://pypi.org/simple"),) + + monkeypatch.setattr(dependency_provenance, "check_release_dependencies", published) install_clean_checkout(monkeypatch, REPO_ROOT, SHA) monkeypatch.setattr(release_publishing, "_api", lambda _root, _repo, endpoint, **_kwargs: responses[endpoint]) monkeypatch.setattr(release_publishing, "_remote_commit", lambda *_args: SHA) @@ -175,7 +185,7 @@ def publish(settings: config.Config, tag: str, *, reuse_evidence: object = None) # irreversible publication boundary with an in-memory request recorder. monkeypatch.setattr(release_publishing, "publish_reviewed_release", publish) # The publication command follows the separately tested notes precheck. - command = shlex.split(dry_run(REPO_ROOT, "release-publish", [TAG]).stderr.splitlines()[-1]) + command = shlex.split(dry_run(REPO_ROOT, "release-publish", [TAG, "/independent-owner"]).stderr.splitlines()[-1]) assert command[:7] == ["uv", "run", "--locked", "--no-sync", "research-repo-tools", "release", "publish"] assert main(["--root", str(REPO_ROOT), *command[5:]]) == 0 assert requested == [TAG] @@ -183,8 +193,8 @@ def publish(settings: config.Config, tag: str, *, reuse_evidence: object = None) def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> None: workflow = yaml.safe_load(WORKFLOW.read_bytes()) - # PyYAML's YAML 1.1 parser reads the Actions `on` key as True. - assert workflow[True] == {"release": {"types": ["published"]}} + assert set(workflow[True]) == {"workflow_dispatch"} + assert set(workflow[True]["workflow_dispatch"]["inputs"]) == {"tag", "candidate", "acceptance"} assert workflow["concurrency"] == {"group": "publish-crates-la-stack", "cancel-in-progress": False} jobs = workflow["jobs"] assert jobs["publish"]["needs"] == "verify" @@ -192,25 +202,28 @@ def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> No assert jobs["verify"]["permissions"] == {"contents": "read", "checks": "read", "actions": "read"} assert jobs["publish"]["permissions"] == {"contents": "read", "checks": "read", "actions": "read", "id-token": "write"} assert "github.repository == 'acgetchell/la-stack'" in jobs["verify"]["if"] - assert jobs["verify"]["steps"][-1]["id"] == "evidence" - assert jobs["verify"]["outputs"] == {"ci-evidence": "${{ steps.evidence.outputs.identity }}"} - assert "CARGO_REGISTRY_TOKEN" not in yaml.safe_dump({"env": workflow["env"], "verify": jobs["verify"]}) + assert "github.triggering_actor == 'acgetchell'" in jobs["verify"]["if"] + assert "CARGO_REGISTRY_TOKEN" not in yaml.safe_dump(jobs["verify"]) for job in jobs.values(): - checkout = job["steps"][0] - assert checkout["with"] == {"ref": "${{ github.sha }}", "fetch-depth": 0, "persist-credentials": False} + assert job["steps"][0]["with"] == {"ref": "${{ github.sha }}", "fetch-depth": 0, "persist-credentials": False} assert job["steps"][1]["with"]["cache"] == "false" + gate = next(step for step in job["steps"] if "just release-receive" in step.get("run", "")) + assert gate["env"]["CANDIDATE_TRANSFER"] == "${{ inputs.candidate }}" + assert gate["env"]["CANDIDATE_ACCEPTANCE"] == "${{ inputs.acceptance }}" + assert 'just release-receive "$RUNNER_TEMP/accepted-candidate" --published' in gate["run"] + assert 'just _release-notes-check "$RELEASE_TAG"' in gate["run"] steps = jobs["publish"]["steps"] - gate = steps[2]["run"] - assert 'just release-gate "$RELEASE_TAG"' in gate - assert 'test -n "$CI_EVIDENCE"' in gate - assert 'just release-evidence --expected "$RUNNER_TEMP/approved-evidence.json"' in gate - assert steps[2]["env"]["CI_EVIDENCE"] == "${{ needs.verify.outputs.ci-evidence }}" - assert 'release registry "$RELEASE_TAG" --expect absent' in gate - assert steps[3]["id"] == "auth" - assert steps[4]["run"].endswith("cargo publish --locked --package la-stack --registry crates-io --no-verify") - assert [index for index, step in enumerate(steps) if "CARGO_REGISTRY_TOKEN" in step.get("env", {})] == [4] - assert steps[5]["if"] == "${{ !cancelled() && steps.auth.outcome == 'success' }}" - assert steps[5]["run"] == 'just release-verify "$RELEASE_TAG"' + assert jobs["verify"]["steps"][3]["id"] == "approval" + assert jobs["verify"]["steps"][3]["with"]["path"].endswith("/acceptance.json") + assert steps[2]["env"]["ARTIFACT_DIGEST"] == "${{ needs.verify.outputs.artifact-digest }}" + assert steps[3]["with"]["artifact-ids"] == "${{ needs.verify.outputs.artifact-id }}" + assert '--expected "$RUNNER_TEMP/approved-acceptance/acceptance.json"' in steps[4]["run"] + assert 'release registry "$RELEASE_TAG" --expect absent' in steps[4]["run"] + assert steps[5]["id"] == "auth" + assert steps[6]["run"].endswith("cargo publish --locked --package la-stack --registry crates-io --no-verify") + assert [index for index, step in enumerate(steps) if "CARGO_REGISTRY_TOKEN" in step.get("env", {})] == [6] + assert steps[7]["if"] == "${{ !cancelled() && steps.auth.outcome == 'success' }}" + assert steps[7]["run"] == 'just release-verify "$RELEASE_TAG"' assert "continue-on-error" not in WORKFLOW.read_text(encoding="utf-8") commands = dry_run(REPO_ROOT, "release-package").stderr assert commands.count("--package la-stack --registry crates-io --dry-run") == 2 @@ -254,8 +267,11 @@ def run_evidence_fixture() -> int: def evidence_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Keep real workflow shell, Just, uv, Python, verifier and output writer.""" source = (REPO_ROOT / "justfile").read_text(encoding="utf-8") - runtime = '[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.6", "python", "-I")]' - assert source.count(runtime) == 2 + runtime = ( + '[script("uvx", "--python", "3.15", "--from", "research-repo-tools @ ' + 'git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e", "python", "-I")]' + ) + assert source.count(runtime) == 3 # Run the exact recipe body/argument transport with the locked test runtime. # The separate isolated-runtime smoke and pin test own uvx installation. source = source.replace(runtime, '[script("uv", "run", "--locked", "--no-sync", "python", "fixture-bootstrap.py")]') @@ -293,8 +309,7 @@ def run_evidence_shell(script: str, root: Path) -> subprocess.CompletedProcess[s def test_evidence_export_uses_real_just_and_installed_writer_without_stdout_capture( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, quiet: bool, outcome: str ) -> None: - workflow = yaml.safe_load(WORKFLOW.read_bytes()) - script = workflow["jobs"]["verify"]["steps"][-1]["run"] + script = 'just release-native-evidence --github-output "$GITHUB_OUTPUT"\n' output = tmp_path / "github output" original = b"prior=value\r\n" if outcome == "directory": @@ -362,8 +377,7 @@ def test_evidence_bootstrap_keeps_windows_import_paths_as_environment_data( output = tmp_path / "github output" output.touch() monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) - workflow = yaml.safe_load(WORKFLOW.read_bytes()) - script = workflow["jobs"]["verify"]["steps"][-1]["run"] + script = 'just release-native-evidence --github-output "$GITHUB_OUTPUT"\n' result = run_evidence_shell(script, tmp_path) assert result.returncode == 0, result.stderr assert result.stdout == "fixture stdout diagnostic" + newline + "Saved verified release evidence." + os.linesep @@ -383,12 +397,11 @@ def test_evidence_bootstrap_keeps_windows_import_paths_as_environment_data( def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, quiet: bool, pin: str ) -> None: - workflow = yaml.safe_load(WORKFLOW.read_bytes()) output = tmp_path / "github output" output.touch() monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) monkeypatch.setenv("JUST_QUIET", str(quiet).lower()) - exported = run_evidence_shell(workflow["jobs"]["verify"]["steps"][-1]["run"], tmp_path) + exported = run_evidence_shell('just release-native-evidence --github-output "$GITHUB_OUTPUT"\n', tmp_path) assert exported.returncode == 0, exported.stderr report = tmp_path / "verifier-result.json" assert json.loads(report.read_text(encoding="utf-8"))["status"] == 0 @@ -405,7 +418,13 @@ def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token # Keep evidence verification real and stop before any credentials or upload. runner = f'uv() {{\n [[ "$4 $5 $6" == "research-repo-tools release registry" ]] || return 90\n touch {shlex.quote(registry.as_posix())}\n}}\n' runner += 'just() { if [[ "$1" == release-gate ]]; then return 0; fi; command just "$@"; }\n' - script = workflow["jobs"]["publish"]["steps"][2]["run"] + script = ( + "set -euo pipefail\n" + 'test -n "$CI_EVIDENCE"\n' + 'printf "%s\\n" "$CI_EVIDENCE" > "$RUNNER_TEMP/approved-evidence.json"\n' + 'just release-native-evidence --expected "$RUNNER_TEMP/approved-evidence.json"\n' + 'uv run --locked --no-sync research-repo-tools release registry "$RELEASE_TAG" --expect absent\n' + ) result = run_evidence_shell(runner + script + f"\ntouch {shlex.quote(token.as_posix())}", tmp_path) if pin != "empty": assert report.exists(), result.stderr @@ -435,7 +454,7 @@ def test_explicit_empty_identity_cannot_select_fresh_evidence(tmp_path: Path, mo monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) expected = tmp_path / "empty.json" expected.write_bytes(b"") - result = run_evidence_shell('just release-evidence --expected empty.json --github-output "$GITHUB_OUTPUT"\n', tmp_path) + result = run_evidence_shell('just release-native-evidence --expected empty.json --github-output "$GITHUB_OUTPUT"\n', tmp_path) observed = json.loads((tmp_path / "verifier-result.json").read_text(encoding="utf-8")) assert observed["arguments"][:2] == ["--expected", "empty.json"] assert result.returncode == observed["status"] == 1 @@ -443,14 +462,16 @@ def test_explicit_empty_identity_cannot_select_fresh_evidence(tmp_path: Path, mo assert output.read_bytes() == original -@pytest.mark.parametrize(("failure", "expected_exit"), [("none", 0), ("evidence", 1), ("ci-evidence", 1), ("empty-pin", 1), ("present", 1), ("unavailable", 2)]) +@pytest.mark.parametrize(("failure", "expected_exit"), [("none", 0), ("evidence", 1), ("candidate", 1), ("present", 1), ("unavailable", 2)]) def test_actual_preupload_shell_requires_evidence_and_proven_absence(tmp_path: Path, failure: str, expected_exit: int) -> None: workflow = yaml.safe_load(WORKFLOW.read_bytes()) - script = workflow["jobs"]["publish"]["steps"][2]["run"] + script = next( + step["run"] for step in workflow["jobs"]["publish"]["steps"] if step.get("name") == "Recheck evidence and registry absence after deployment approval" + ) marker = tmp_path / "upload-reached" gate = "just() { return 1; }" if failure == "evidence" else "just() { return 0; }" - if failure == "ci-evidence": - gate = 'just() { [[ "$1" != release-evidence ]]; }' + if failure == "candidate": + gate = 'just() { [[ "$1" != release-receive ]]; }' registry_exit = expected_exit if failure in {"present", "unavailable"} else 0 registry = f"uv() {{ return {registry_exit}; }}" script = f"{gate}\n{registry}\n{script}\ntouch {shlex.quote(marker.as_posix())}" @@ -460,7 +481,7 @@ def test_actual_preupload_shell_requires_evidence_and_proven_absence(tmp_path: P env={ **{key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "SHELLOPTS", "BASHOPTS"}}, "RELEASE_TAG": TAG, - "CI_EVIDENCE": "" if failure == "empty-pin" else '{"run_id": 11}', + "CANDIDATE_TRANSFER": '{"schema": 1}', "RUNNER_TEMP": tmp_path.as_posix(), }, check=False, @@ -469,6 +490,37 @@ def test_actual_preupload_shell_requires_evidence_and_proven_absence(tmp_path: P assert marker.exists() == (failure == "none") +@pytest.mark.parametrize("failure", ["none", "attempt", "identifier", "digest", "expired", "different-digest", "different-run"]) +def test_actual_approval_artifact_guard_rejects_drift(tmp_path: Path, failure: str) -> None: + workflow = yaml.safe_load(WORKFLOW.read_bytes()) + script = next(step["run"] for step in workflow["jobs"]["publish"]["steps"] if step.get("name") == "Require this exact approval receipt") + metadata = { + "expired": failure == "expired", + "digest": "sha256:" + ("b" if failure == "different-digest" else "a") * 64, + "workflow_run": {"id": 99 if failure == "different-run" else 42}, + } + supplied = tmp_path / "artifact metadata.json" + supplied.write_text(json.dumps(metadata), encoding="utf-8", newline="\n") + marker = tmp_path / "approval-reached" + provider = f'gh() {{ [[ "$*" == "api repos/acgetchell/la-stack/actions/artifacts/7" ]] && cat {shlex.quote(supplied.as_posix())}; }}' + result = run_command( + "bash", + ["--noprofile", "--norc", "-euc", f"{provider}\n{script}\ntouch {shlex.quote(marker.as_posix())}"], + env={ + **{key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "SHELLOPTS", "BASHOPTS"}}, + "VALIDATED_ATTEMPT": "1" if failure == "attempt" else "2", + "GITHUB_RUN_ATTEMPT": "2", + "ARTIFACT_ID": "07" if failure == "identifier" else "7", + "ARTIFACT_DIGEST": "invalid" if failure == "digest" else "a" * 64, + "GITHUB_RUN_ID": "42", + "GITHUB_REPOSITORY": "acgetchell/la-stack", + }, + check=False, + ) + assert result.returncode == (0 if failure == "none" else 1), result.stderr + assert marker.exists() == (failure == "none") + + def test_main_ci_seals_only_complete_native_and_cargo_validation() -> None: workflow = yaml.safe_load((REPO_ROOT / ".github/workflows/ci.yml").read_bytes()) jobs = workflow["jobs"] @@ -483,7 +535,19 @@ def test_main_ci_seals_only_complete_native_and_cargo_validation() -> None: assert evidence["permissions"] == {"contents": "read", "actions": "read", "attestations": "write", "id-token": "write"} steps = {step["name"]: step for step in evidence["steps"] if "name" in step} capture = steps["Capture successful native validation identities"]["run"] - assert "uvx --python 3.15 --from research-repo-tools==0.2.6 python" in capture + assert steps["Capture successful native validation identities"]["id"] == "evidence" + for name in ("Sign original main validation evidence", "Retain the signature bundle", "Upload signed validation evidence"): + assert steps[name]["if"] == "steps.evidence.outputs.eligible == 'true'" + assert shlex.split(capture.split("<<'PY'", 1)[0].replace("\\\n", "")) == [ + "uvx", + "--python", + "3.15", + "--from", + "research-repo-tools @ git+https://github.com/acgetchell/research-repo-tools@055353c630604b1ae3438c1eaf534ba85e067a7e", + "python", + "-I", + "-", + ] assert '"release", "evidence", "create", "tooling/release-evidence.toml"' in capture assert '"--output", "target/release-evidence/validation.json"' in capture assert steps["Sign original main validation evidence"]["with"]["subject-path"] == "target/release-evidence/validation.json" @@ -496,6 +560,38 @@ def test_main_ci_seals_only_complete_native_and_cargo_validation() -> None: } +@pytest.mark.parametrize("source", ["development", "published", "rejected"]) +def test_main_ci_creates_no_release_evidence_for_development_pins(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + """Run the actual workflow adapter; only a successful release gate enables signing.""" + workflow = yaml.safe_load((REPO_ROOT / ".github/workflows/ci.yml").read_bytes()) + step = next(item for item in workflow["jobs"]["release-evidence"]["steps"] if item.get("id") == "evidence") + script = tmp_path / "capture.py" + script.write_text(step["run"].split("<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0], encoding="utf-8") + output = tmp_path / "github output" + original = b"prior=value\n" + output.write_bytes(original) + for key, value in {"GITHUB_OUTPUT": str(output), "GITHUB_SHA": SHA, "GITHUB_RUN_ID": "11", "GITHUB_RUN_ATTEMPT": "1"}.items(): + monkeypatch.setenv(key, value) + pin = dependency_provenance.DependencyIdentity("research-repo-tools", "0.2.6", "git", "a" * 40) if source == "development" else None + monkeypatch.setattr(dependency_provenance, "check_development_dependency", lambda _root: pin) + calls: list[list[str]] = [] + + def create(arguments: list[str]) -> int: + calls.append(arguments) + return 1 if source == "rejected" else 0 + + monkeypatch.setattr("research_repo_tools.cli.main", create) + with pytest.raises(SystemExit) as stopped: + runpy.run_path(str(script), run_name="__main__") + assert stopped.value.code == (1 if source == "rejected" else 0) + assert len(calls) == (0 if source == "development" else 1) + if calls: + assert calls[0][:4] == ["release", "evidence", "create", "tooling/release-evidence.toml"] + data = output.read_bytes() + assert data.startswith(original) + assert (b"eligible" in data) == (source == "published") + + def test_native_packaging_failure_stops_before_further_validation(tmp_path: Path) -> None: script = dry_run(REPO_ROOT, "release-package").stderr calls = tmp_path / "calls" diff --git a/tests/tooling/test_toolchain_integration.py b/tests/tooling/test_toolchain_integration.py index 2c9daca..4e28d2d 100644 --- a/tests/tooling/test_toolchain_integration.py +++ b/tests/tooling/test_toolchain_integration.py @@ -7,13 +7,15 @@ import sys import tomllib import zipfile -from importlib.metadata import version +from importlib.metadata import distribution, version from pathlib import Path from typing import TYPE_CHECKING, cast import pytest from packaging.requirements import Requirement from research_repo_tools.cli import main +from research_repo_tools.dependency_policy import load_policy +from research_repo_tools.dependency_provenance import check_development_dependency from research_repo_tools.process import run_command from research_repo_tools.python_tools import TOOLS, versions as shared_tool_versions @@ -21,6 +23,9 @@ import subprocess REPO_ROOT = Path(__file__).resolve().parents[2] +TOOLING_REVISION = "055353c630604b1ae3438c1eaf534ba85e067a7e" +TOOLING_REPOSITORY = "https://github.com/acgetchell/research-repo-tools" +TOOLING_REQUIREMENT = f"research-repo-tools @ git+{TOOLING_REPOSITORY}@{TOOLING_REVISION}" CHECKED = ["run", "--locked", "--no-sync", "--no-python-downloads", "research-repo-tools", "toolchain", "run", "--"] UPDATER = ["run", "--locked", "--only-group", "tooling", "--inexact", "research-repo-tools"] UV_UPDATE = ["run", "--no-config", "--no-sync", "--no-python-downloads", "research-repo-tools", "deps", "update-uv"] @@ -44,8 +49,8 @@ } -def test_published_tooling_package_matches_manifest_lock_and_installation() -> None: - """Keep the exact published-package contract in one consumer test.""" +def test_tooling_commit_matches_manifest_lock_and_installation() -> None: + """Distinguish the reviewed upstream commit from its unchanged version label.""" expected_version = "0.2.6" manifest = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) assert manifest["project"]["dependencies"] == [] @@ -59,42 +64,94 @@ def test_published_tooling_package_matches_manifest_lock_and_installation() -> N lock = tomllib.loads((REPO_ROOT / "uv.lock").read_text(encoding="utf-8")) package = next(item for item in lock["package"] if item["name"] == "research-repo-tools") assert package["version"] == expected_version - assert package["source"] == {"registry": "https://pypi.org/simple"} - assert package["wheels"][0]["hash"] == "sha256:89ebae476b5bde0cffc321603ed1d867796e3276a7fb88c7dc1f7de53a13e7c2" - assert package["sdist"]["hash"] == "sha256:dd826a916f037f47b925ca3268adc76d415a8561547bedda348a67f4f2e2e065" + assert manifest["tool"]["uv"]["sources"]["research-repo-tools"] == {"git": TOOLING_REPOSITORY, "rev": TOOLING_REVISION} + assert package["source"] == {"git": f"{TOOLING_REPOSITORY}?rev={TOOLING_REVISION}#{TOOLING_REVISION}"} + # Distribution.read_text decodes UTF-8 internally and has no encoding argument. + installed = json.loads( + # nosemgrep: la-stack.python.explicit-path-text-encoding-in-tests + distribution("research-repo-tools").read_text("direct_url.json") or "{}" + ) + assert installed["url"] == TOOLING_REPOSITORY + assert installed["vcs_info"] == {"vcs": "git", "commit_id": TOOLING_REVISION, "requested_revision": TOOLING_REVISION} locked = {item["name"]: item["version"] for item in lock["package"]} for name, expected in shared_tool_versions().items(): assert locked[name] == version(name) == expected + identity = check_development_dependency(REPO_ROOT) + assert identity is not None + assert (identity.name, identity.version, identity.revision) == ("research-repo-tools", expected_version, TOOLING_REVISION) + + +def test_release_dependency_policy_covers_tooling_and_private_comparison_workspace() -> None: + policy = load_policy(REPO_ROOT) + assert policy is not None + assert {(project.ecosystem, project.directory, project.owned) for project in policy.projects} == { + ("python", ".", (".",)), + ("cargo", ".", (".", "benches/comparison")), + } + manifest = tomllib.loads((REPO_ROOT / "Cargo.toml").read_text(encoding="utf-8")) + assert manifest["workspace"]["members"] == ["benches/comparison"] + comparison = tomllib.loads((REPO_ROOT / "benches/comparison/Cargo.toml").read_text(encoding="utf-8")) + assert comparison["package"]["publish"] is False -def test_isolated_release_runtime_uses_the_published_pin_outside_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize( + "command", + [ + ["release", "dependencies"], + ["performance", "prepare", "tooling/performance-release.toml", "--release-intent"], + ["performance", "release", "tooling/performance-release.toml", "--release-intent"], + ], +) +def test_development_pin_cannot_enter_release_work(command: list[str], tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """The installed public gate rejects before reading plans or creating evidence.""" + for name in ("pyproject.toml", "uv.lock", "dependency-sources.toml"): + shutil.copyfile(REPO_ROOT / name, tmp_path / name) + before = {path.name: path.read_bytes() for path in tmp_path.iterdir()} + assert main(["--root", str(tmp_path), *command]) == 1 + diagnostic = capsys.readouterr().err + assert "release requires a published registry dependency without tool.uv.sources overrides" in diagnostic + assert "Publish the supplying upstream version" in diagnostic + assert {path.name: path.read_bytes() for path in tmp_path.iterdir()} == before + + +def test_isolated_release_runtime_uses_the_same_commit_outside_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Exercise the actual trusted runtime on each native CI platform.""" expected = version("research-repo-tools") source = (REPO_ROOT / "justfile").read_text(encoding="utf-8") - assert source.count(f'[script("uvx", "--python", "3.15", "--from", "research-repo-tools=={expected}", "python", "-I")]') == 2 + assert source.count(f'[script("uvx", "--python", "3.15", "--from", "{TOOLING_REQUIREMENT}", "python", "-I")]') == 3 (tmp_path / "research_repo_tools.py").write_text('raise RuntimeError("untrusted checkout import")\n', encoding="utf-8") + (tmp_path / "tooling").mkdir() + (tmp_path / "justfile").write_bytes((REPO_ROOT / "justfile").read_bytes()) + (tmp_path / "tooling/release_candidate.py").write_text( + "import json, sys, research_repo_tools\nfrom importlib.metadata import distribution\n" + 'd = distribution("research-repo-tools")\n' + 'print(json.dumps([d.version, research_repo_tools.__file__, json.loads(d.read_text("direct_url.json")), sys.argv[1:]]))\n', + encoding="utf-8", + newline="\n", + ) monkeypatch.setenv("PYTHONPATH", str(tmp_path)) result = run_command( - "uvx", + "just", [ - "--python", - "3.15", - "--from", - f"research-repo-tools=={expected}", - "python", - "-I", - "-c", - ( - "import json, research_repo_tools; from importlib.metadata import version; " - 'print(json.dumps([version("research-repo-tools"), research_repo_tools.__file__]))' - ), + "--no-deps", + "--justfile", + str(REPO_ROOT / "justfile"), + "--working-directory", + str(tmp_path), + "release-receive", + str(tmp_path / "output with spaces"), + "--incoming", + str(tmp_path / "input with spaces"), ], timeout=180, cwd=tmp_path, ) - installed_version, location = json.loads(result.stdout) + installed_version, location, provenance, arguments = json.loads(result.stdout) assert installed_version == expected + assert provenance["vcs_info"]["commit_id"] == TOOLING_REVISION assert not Path(location).resolve().is_relative_to(REPO_ROOT) + assert not Path(location).resolve().is_relative_to(tmp_path) + assert arguments == ["receive", str(tmp_path / "output with spaces"), "--incoming", str(tmp_path / "input with spaces")] @pytest.fixture @@ -150,6 +207,36 @@ def test_update_failure_stops_later_steps(consumer: Path, monkeypatch: pytest.Mo assert calls == WORKFLOWS["update"][:step] +@pytest.mark.parametrize("workers", [None, "0", "2"]) +@pytest.mark.parametrize("failure", [False, True]) +def test_python_recipe_preserves_worker_selection_and_failure(consumer: Path, monkeypatch: pytest.MonkeyPatch, workers: str | None, failure: bool) -> None: + if workers is None: + monkeypatch.delenv("LA_STACK_TEST_WORKERS", raising=False) + else: + monkeypatch.setenv("LA_STACK_TEST_WORKERS", workers) + if failure: + monkeypatch.setenv("UPDATE_FAIL_STEP", "4") + + result, calls = invoke(consumer, "test-python") + + assert result.returncode == (23 if failure else 0), result.stderr + assert len(calls) == 4 + invocation = calls[-1] + assert invocation[invocation.index("pytest") :] == [ + "pytest", + "-q", + "-n", + workers or "4", + "--dist", + "worksteal", + "--max-worker-restart", + "0", + "--durations=20", + ] + if failure: + assert "fixture update failed" in result.stderr + + def test_declared_tools_are_verified_and_cargo_edit_runs_from_managed_store(capsys: pytest.CaptureFixture[str]) -> None: """Exercise the installed public CLI on each native CI platform, without installs.""" assert main(["--root", str(REPO_ROOT), "toolchain", "check", "--json"]) == 0 @@ -198,8 +285,12 @@ def test_python_update_preserves_shared_pin_and_tools_with_explicit_dev_sync(tmp registry = tmp_path / "wheels" registry.mkdir() original = (REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8") - # Empty fixture wheels represent resolver candidates, not executable tools. - source = original.replace("package = false", "package = false\ndefault-groups = []") + # This fixture tests registry updates against inert local wheels. The + # development Git source has separate identity coverage above and cannot + # resolve in the fixture's deliberately empty offline cache. + override = f'[tool.uv.sources]\nresearch-repo-tools = {{ git = "{TOOLING_REPOSITORY}", rev = "{TOOLING_REVISION}" }}\n' + assert original.count(override) == 1 + source = original.replace(override, "").replace("package = false", "package = false\ndefault-groups = []") manifest = root / "pyproject.toml" manifest.write_text(source, encoding="utf-8") before = tomllib.loads(source) diff --git a/tooling/README.md b/tooling/README.md index 0c36846..6a2d7b3 100644 --- a/tooling/README.md +++ b/tooling/README.md @@ -1,8 +1,8 @@ # Tooling The installed `research-repo-tools` package owns common development and -performance tooling. The exact released pin and registry hashes live in -`pyproject.toml` and `uv.lock`. This dependency-only environment has no local +performance tooling. The exact supplier pin lives in `pyproject.toml` and +`uv.lock`. This dependency-only environment has no local Python distribution or console entry points. Declarative release policies select the native Cargo gates; shared tooling is not copied into this repository. @@ -17,7 +17,11 @@ versions. `toolchain.inherit-python-tools` verifies declarations, the locked development graph and executable versions. Local lint rules remain in this repository; the `dev` group does not carry competing tool pins. `just test-python` -runs the installed-package integration tests. `just ci` is the final +runs every installed-package integration test with four isolated pytest workers, +work-stealing scheduling and no worker restarts. Set `LA_STACK_TEST_WORKERS=0` +for serial diagnosis or another fixed count to bound local resource use. The +slowest test durations are reported; assertions and test selection are unchanged. +`just ci` is the final repository validation. Native Linux, macOS and Windows CI is required for adoption; local macOS results do not establish the other platforms. @@ -32,7 +36,7 @@ exit alone also accepts a broken import or shell. When quiet mode hides diagnostics, retain the verifier's arguments, exit status and diagnostic in a fixture-owned result file. -The `release-evidence` recipe uses Just's native uv/Python script interpreter. +The candidate and native evidence recipes use Just's isolated uvx/Python script interpreter. MSYS applies shell-like argument expansion when started by a native Windows program, which can corrupt JSON passed as Bash positional arguments; see [MSYS2 #522](https://github.com/msys2/MSYS2-packages/issues/522). The integration @@ -54,9 +58,38 @@ Other platforms resolve Semgrep 1.184.0 or newer within the shared range. ## Shared capability adoption -The capability inventory below uses the installed v0.2.6 CLI and -its [versioned contracts](https://github.com/acgetchell/research-repo-tools/blob/v0.2.6/README.md#common-workflows). -Trusted release jobs use the same exact package pin through `uvx --python 3.15`. +This development branch tests upstream commit +`055353c630604b1ae3438c1eaf534ba85e067a7e` through an exact uv Git source override. +The package metadata still says 0.2.6; integration checks verify the manifest, +lock and installed commit, including the isolated `uvx --python 3.15` runtimes. +Advance the full commit deliberately after upstream changes; do not float on +`main`. Before an actual release, replace the override and isolated runtime pins +with the supplying published version, resolve/sync the lock and revalidate all +affected evidence. The configured `release.published-dependencies` policy uses +the shared implementation from +[research-repo-tools#193](https://github.com/acgetchell/research-repo-tools/issues/193). +`just release-dependencies` checks that transition before release work. It +deliberately rejects this development pin; ordinary checks and bounded +acquisition remain available. Release evidence and publication enforce the +same policy again, including the actual isolated runtime. + +Normal `just update` uses the supplier's exact Git-source support from +[research-repo-tools#201](https://github.com/acgetchell/research-repo-tools/pull/201), +which fixes [#200](https://github.com/acgetchell/research-repo-tools/issues/200). +The updater preserves the included tooling group, immutable source commit and +inherited tool versions while updating unrelated dependencies. Keep the source +mapping throughout maintenance; release operations still require publication. + +The root `dependency-sources.toml` declares both dependency graphs for the +shared source-policy validation introduced by +[research-repo-tools#198](https://github.com/acgetchell/research-repo-tools/pull/198). +Cargo release ownership includes `la-stack` and its private comparison member; +the Python project is the private tooling environment. Other dependencies must +come from their registries for release. The file is retained in source exports +and both benchmark harness declarations, so policy changes invalidate evidence. + +The capability inventory below uses the pinned CLI and +its [exact-commit contracts](https://github.com/acgetchell/research-repo-tools/blob/055353c630604b1ae3438c1eaf534ba85e067a7e/README.md#common-workflows). The shared updater installs and verifies candidate Cargo tools before publishing new pins; no local updater implementation is needed. Native Rust algorithms, assertions, benchmark selection and release decisions @@ -106,8 +139,9 @@ external dependencies and commit pins. The [benchmark guide](../docs/BENCHMARKING.md) owns command selection, interpretation and output paths. Measurement and publication use the installed -CLI or native Cargo commands. `performance-release.toml` composes the installed -`performance prepare`, `release` and `readme` commands. Preparation retains +APIs or native Cargo commands. `performance-release.toml` declares preparation, +release measurement and README publication. `release_candidate.py` supplies +repository orchestration for independent owner records and hosted transport. Preparation retains independent commit assertions and declaration identity; measurement/report promotion and README publication remain separate retry boundaries. There is no runtime configuration generator. @@ -117,44 +151,84 @@ There is no runtime configuration generator. | `measurements/full.toml`, `measurements/non-exact.toml` | Release floor, required cases, sampling, original peer phases, source/harness inventories, probes and environments | | `performance-policy.toml` | Independent scientific catalog used by consumer acceptance tests | | `performance.just` | Native discovery, correctness gates and measurement commands | -| `criterion-inventory.toml` | Hosted la-stack-only discovery and independent required scientific IDs | +| `criterion-inventory.toml` | Optional saved-baseline discovery and independent required scientific IDs | | `criterion-saved.toml` | Required IDs, canonical paths, 100 samples and exact saved raw JSON bytes | | `performance-report.toml` | Independent release eligibility, current report and immutable run history | | `performance-release.toml` | Preparation storage, publication chronology and separate phase declarations | +| `release_candidate.py` | Owner-controlled capture, final acceptance and exact hosted handoffs using public shared APIs | | `performance-readme.toml` | Canonical README selection, formatter, source freshness and tagged links | | `plots/{non-exact,retained,scratch}/*.toml` | Named metric/statistic/axis selections with explicit input ownership | Both labels of a new measurement must be v0.5.0 or newer. The full and non-exact declarations require 100 samples, mean and median estimates, 95% intervals and nanoseconds. Each phase runs its native correctness gate before -timing. The current phase's `gate-public` adds column bases, rational dot products, -interval solve oracles and polynomial coefficient checks to the shared `gate`. -The baseline uses `gate` because those APIs are absent before v0.5.2. -Reference libraries originate in the baseline phase. +timing. Schema-6 availability resolves each suite before compilation, inside one +sequential trial. README and LU plots select trial 1. New-API +suites run their column-basis, rational-dot, interval-solve or polynomial gate +on every supported revision. The non-exact diagnostic retains its original gate +and baseline-phase reference libraries. The expanded #301 policy measures all public la-stack workflows at the current -revision and the 1,254 shared cases at the baseline, plus only 16 baseline-phase -peer LU-factorization-and-solve cases. The 149 cases for APIs introduced in v0.5.2 -retain current timings without baseline comparisons. Hosted -acquisition omits peers entirely. The full scratch/retained plot selections +revision and on supported baselines. Older baselines retain 1,254 shared library +cases; the 149 cases introduced in v0.5.2 become required automatically at that +boundary. The paired LU suite acquires 16 peer cases on each phase alongside +the matching library cases; README ratios use current-phase observations. Hosted +release jobs reuse the accepted local acquisition without collecting timings. The full scratch/retained plot selections therefore expose only LU solve; broad peer plots remain in the explicitly -requested non-exact diagnostic. `performance.just` prints the independent local -case budget before measurement, and `performance-release` records the recipe +requested non-exact diagnostic. The shared reporter prints the resolved suite +inventory and nominal budget before measurement, and `performance-release` records the recipe body's elapsed seconds and failure status under ignored scratch outputs. The [public workflow map](../docs/BENCHMARKING.md#release-acquisition-policy) owns coverage decisions, feature configurations and runtime estimates. +`performance-release` forwards its `progress` and `cadence` arguments directly +to the installed reporter, defaulting to `human` and `300` seconds. JSON status +and `none` are supported without a local progress engine. Consumer fixtures +exercise validated totals, unknown ETA, captured logs and early failure; report +and README recipes retain their measurement-free retry boundaries. The +[workload review](../docs/dev/benchmark-workload-review.md) records the availability +decision, preserved coverage and supplier's corrected gate-failure event. + General workload availability across API revisions is tracked in [research-repo-tools#166](https://github.com/acgetchell/research-repo-tools/issues/166) for v0.2.5, with published-package adoption in [la-stack#320](https://github.com/acgetchell/la-stack/issues/320) for v0.5.3. -Keep the explicit v0.5.2 baseline boundary until that replacement is verified. +The full declaration now uses inclusive `since = "v0.5.2"` suite bounds. +The normally installed 0.2.6 interface was checked against the consumer's peer +comparison contract. Astra recommends retaining its cross-phase different-ID +restriction and migrating to paired LU suites with current-phase peer ratios. +The current development pin fixes the gate-status defect tracked in +[research-repo-tools#191](https://github.com/acgetchell/research-repo-tools/issues/191); +the preserved schema-2 and schema-5 regressions pass against it. +No permanent baseline exclusion remains; supported missing cases or failed +compilation remain errors. + +The current pin also supplies schema-6 interrupted-trial recovery and +phase-specific sources from [PR #205](https://github.com/acgetchell/research-repo-tools/pull/205). +Baseline sources declare historical README compiler input; current gates prove +README is excluded from ordinary debug/release compilation. Current README is +one of the precise generated outputs. Historical current revisions that still +compile it are rejected by the full candidate plan. + +Earlier adopted fixes for +[research-repo-tools#196](https://github.com/acgetchell/research-repo-tools/issues/196) +and [#197](https://github.com/acgetchell/research-repo-tools/issues/197) arrived through +[PR #199](https://github.com/acgetchell/research-repo-tools/pull/199). +The paired-suite publisher resolves dependency-version labels from the owning suite +observation; preparation checks each available suite's minimum release before +saving a source pair. The preserved public-API reproducers pass without changing +their assertions. The [workload review](../docs/dev/benchmark-workload-review.md#validation-evidence) +records downstream validation and the remaining publication/platform boundary. Working-tree measurements include tracked changes and nonignored untracked regular files. Scratch outputs belong under ignored `target/`. `just performance-release` fetches tags and saves independent source assertions before measurement and report promotion. It stops if preparation fails; -`performance-readme` remains a separate publication step. +`performance-readme` remains a separate publication step. `performance-resume` +reuses the same preparation and checkpoint; it preserves completed suite pairs +inside an interrupted trial. A fresh acquisition refuses an existing checkpoint. +The owner record lives outside the checkout and freezes original bytes/digests +before promotion. Explicit capture retries never start measurements. For preparation alone, `just performance-prepare` uses local tags; `--fetch-tags --allow-git-mutations` explicitly enables fetching. Measurement recipes create temporary @@ -189,9 +263,10 @@ Shared implementation matrices live upstream, including `public_criterion_consumer.py`, exercised from wheel and sdist installations. The shared package also owns `public_suites_consumer.py`, which covers suite selection, raw-file digests, archive paths, original host retention and complete assembly failures. -The release workflow calls `performance pack-suite` and `assemble-suites` -directly; `bench-release-check-suite` is a diagnostic wrapper around -`performance verify-saved --suite`. No local suite verifier or merge engine remains. +Legacy suite archive consumer fixtures retain public `pack-suite` and +`assemble-suites` coverage; `bench-release-check-suite` remains diagnostic. +New releases derive archives from the original candidate through shared APIs. +There is no local suite verifier, comparison engine or archive merger. Consumer tests retain independent scientific selections, actual recipe wiring, saved archive round trips, publication boundaries and historical hashes. @@ -213,22 +288,32 @@ and scientific checks on the exact commit, and requires the durable Criterion release asset. `release-gate`, `release-publish` and `release-verify` use the shared release contract; `release-package` owns native default/exact Cargo dry-runs. Main CI signs those packaging and native validation identities; -`release-evidence` verifies the original main-push attempt and immutable signed -artifact before publication and rechecks its pinned identity after approval. +`release-evidence ` binds the original acquisition, precise publisher +receipt, final clean commit and signed native acceptance. Named review records +are immutable; selecting a later native attempt requires an explicit new name. `release-evidence.toml` owns the package, workflow, native job and signer policy; -`release-assets.toml` owns the draft-target policy. Both recipes use the pinned +`release-assets.toml` owns the draft-target policy. Evidence recipes use the pinned installed runtime outside the observed checkout. The [release guide](../docs/RELEASING.md#reusable-ci-evidence) -owns the transactional output and frozen approval contract. +owns the independent owner selection and frozen approval contract. +`release-stage` freezes the draft ID before upload; `release-publication` adds +canonical archive ID/hash assertions. Two maintainer dispatches use those +independent records. Write-scope hosted jobs execute no checkout code; +read-only verification authenticates full acceptance with an owner-reviewed +SHA-256 and freezes its bytes in an immutable Actions artifact across approval. +Explicit native-evidence renewal changes that reviewed receipt while retaining +the original immutable release archive and asset selections. +`tests/tooling/test_candidate_transfer.py` exercises the actual handoff blocks. `tests/tooling/test_release_evidence.py` checks installed policy boundaries, while `tests/tooling/test_release_publishing.py` checks consumer policy and upload barriers without registry credentials. The release guide owns retention, complete-rerun recovery and the distinction between reused checks and Cargo repackaging. `tests/tooling/test_toolchain_integration.py` owns the package-identity check: -manifest pin, registry lock, installed version and distribution hashes. +manifest source, exact Git lock and installed commit for this rehearsal. Dependency updates and release tests derive their requirement from the manifest. The shared package pin advances deliberately with its lockfile; -no sibling checkout supplies runtime code. +no sibling checkout supplies runtime code. A published release will replace the +temporary source override before consumer publication. Setup, managed execution, dependency updates, Markdown checks, Semgrep, security scans and opt-in CodeRabbit review also use shared commands. diff --git a/tooling/measurements/full.toml b/tooling/measurements/full.toml index e7a55e3..2e23fda 100644 --- a/tooling/measurements/full.toml +++ b/tooling/measurements/full.toml @@ -1,5 +1,28 @@ -# Scientific measurement policy consumed directly by research-repo-tools. -schema = 2 +# Scientific acquisition units; availability resolves before native compilation. +schema = 6 +outputs = [ + "README.md", + "docs/assets/bench/vs_linalg_lu_solve_mean_linear.csv", + "docs/assets/bench/vs_linalg_lu_solve_mean_linear.svg", + "docs/assets/bench/vs_linalg_lu_solve_mean_log.csv", + "docs/assets/bench/vs_linalg_lu_solve_mean_log.svg", + "docs/assets/bench/vs_linalg_lu_solve_median_linear.csv", + "docs/assets/bench/vs_linalg_lu_solve_median_linear.svg", + "docs/assets/bench/vs_linalg_lu_solve_median_log.csv", + "docs/assets/bench/vs_linalg_lu_solve_median_log.svg", +] + +[schedule] +trials = 1 +order = "sequential" + +[[suites]] +name = "core-default" +contract = "la-stack-core-default-v1" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 minimum-release = "v0.5.0" sources = [ "src/**/*.rs", "Cargo.toml" ] harness = [ @@ -8,6 +31,7 @@ harness = [ "examples/**/*.rs", "Cargo.toml", "Cargo.lock", + "dependency-sources.toml", "rust-toolchain.toml", "pyproject.toml", "uv.lock", @@ -20,6 +44,8 @@ harness = [ "tests/common/proptest_config.rs", "tests/polynomial_bench_inputs.rs", "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", "tooling/criterion-inventory.toml", "tooling/performance-policy.toml", "tooling/measurements/*.toml", @@ -41,29 +67,46 @@ compatible = [ "context.dependency.nalgebra", "context.dependency.faer", ] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] -[context] +[suites.measurement.context] phase-driver = "just-v1" -suite = "all" -scope = "public-release-and-lu-peers-v3" -baseline-api-compatibility = "pre-v0.5.2-public-api" +suite = "core-default" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" -[probes] +[suites.measurement.probes] rustc = [ "rustc", "-vV" ] cargo = [ "cargo", "--version" ] just = [ "just", "--version" ] -[dependencies] +[suites.measurement.dependencies] criterion = "Cargo.lock" nalgebra = "Cargo.lock" faer = "Cargo.lock" -[phases.baseline] -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-all" ] +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "core-default" ] gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate" ] -environment = { CARGO_TARGET_DIR = "target", CRITERION_HOME = "target/criterion", RUSTUP_TOOLCHAIN = "1.99.1", RUSTFLAGS = "--cap-lints=warn", CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn", RUSTC = "rustc", RUSTC_WRAPPER = "", RUSTC_WORKSPACE_WRAPPER = "" } -[phases.baseline.discovery] +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] required = [ "angle_d3/dense/borrowed_slice", "angle_d3/dense/construct_then_angle", @@ -233,135 +276,6 @@ required = [ "angle_d6/subnormal_axes/borrowed_slice", "angle_d6/subnormal_axes/construct_then_angle", "angle_d6/subnormal_axes/prepared_vector", - "canonical_conversion_below_overflow_d2/rounded_array", - "canonical_conversion_below_overflow_d2/rounded_result", - "canonical_conversion_below_overflow_d2/strict_array", - "canonical_conversion_below_overflow_d2/strict_result", - "canonical_conversion_below_overflow_d3/rounded_array", - "canonical_conversion_below_overflow_d3/rounded_result", - "canonical_conversion_below_overflow_d3/strict_array", - "canonical_conversion_below_overflow_d3/strict_result", - "canonical_conversion_below_overflow_d4/rounded_array", - "canonical_conversion_below_overflow_d4/rounded_result", - "canonical_conversion_below_overflow_d4/strict_array", - "canonical_conversion_below_overflow_d4/strict_result", - "canonical_conversion_below_overflow_d5/rounded_array", - "canonical_conversion_below_overflow_d5/rounded_result", - "canonical_conversion_below_overflow_d5/strict_array", - "canonical_conversion_below_overflow_d5/strict_result", - "canonical_conversion_dyadic_d2/rounded_array", - "canonical_conversion_dyadic_d2/rounded_result", - "canonical_conversion_dyadic_d2/strict_array", - "canonical_conversion_dyadic_d2/strict_result", - "canonical_conversion_dyadic_d3/rounded_array", - "canonical_conversion_dyadic_d3/rounded_result", - "canonical_conversion_dyadic_d3/strict_array", - "canonical_conversion_dyadic_d3/strict_result", - "canonical_conversion_dyadic_d4/rounded_array", - "canonical_conversion_dyadic_d4/rounded_result", - "canonical_conversion_dyadic_d4/strict_array", - "canonical_conversion_dyadic_d4/strict_result", - "canonical_conversion_dyadic_d5/rounded_array", - "canonical_conversion_dyadic_d5/rounded_result", - "canonical_conversion_dyadic_d5/strict_array", - "canonical_conversion_dyadic_d5/strict_result", - "canonical_conversion_min_subnormal_d2/rounded_array", - "canonical_conversion_min_subnormal_d2/rounded_result", - "canonical_conversion_min_subnormal_d2/strict_array", - "canonical_conversion_min_subnormal_d2/strict_result", - "canonical_conversion_min_subnormal_d3/rounded_array", - "canonical_conversion_min_subnormal_d3/rounded_result", - "canonical_conversion_min_subnormal_d3/strict_array", - "canonical_conversion_min_subnormal_d3/strict_result", - "canonical_conversion_min_subnormal_d4/rounded_array", - "canonical_conversion_min_subnormal_d4/rounded_result", - "canonical_conversion_min_subnormal_d4/strict_array", - "canonical_conversion_min_subnormal_d4/strict_result", - "canonical_conversion_min_subnormal_d5/rounded_array", - "canonical_conversion_min_subnormal_d5/rounded_result", - "canonical_conversion_min_subnormal_d5/strict_array", - "canonical_conversion_min_subnormal_d5/strict_result", - "canonical_conversion_negative_underflow_d2/rounded_array", - "canonical_conversion_negative_underflow_d2/rounded_result", - "canonical_conversion_negative_underflow_d2/strict_array", - "canonical_conversion_negative_underflow_d2/strict_result", - "canonical_conversion_negative_underflow_d3/rounded_array", - "canonical_conversion_negative_underflow_d3/rounded_result", - "canonical_conversion_negative_underflow_d3/strict_array", - "canonical_conversion_negative_underflow_d3/strict_result", - "canonical_conversion_negative_underflow_d4/rounded_array", - "canonical_conversion_negative_underflow_d4/rounded_result", - "canonical_conversion_negative_underflow_d4/strict_array", - "canonical_conversion_negative_underflow_d4/strict_result", - "canonical_conversion_negative_underflow_d5/rounded_array", - "canonical_conversion_negative_underflow_d5/rounded_result", - "canonical_conversion_negative_underflow_d5/strict_array", - "canonical_conversion_negative_underflow_d5/strict_result", - "canonical_conversion_non_dyadic_d2/rounded_array", - "canonical_conversion_non_dyadic_d2/rounded_result", - "canonical_conversion_non_dyadic_d2/strict_array", - "canonical_conversion_non_dyadic_d2/strict_result", - "canonical_conversion_non_dyadic_d3/rounded_array", - "canonical_conversion_non_dyadic_d3/rounded_result", - "canonical_conversion_non_dyadic_d3/strict_array", - "canonical_conversion_non_dyadic_d3/strict_result", - "canonical_conversion_non_dyadic_d4/rounded_array", - "canonical_conversion_non_dyadic_d4/rounded_result", - "canonical_conversion_non_dyadic_d4/strict_array", - "canonical_conversion_non_dyadic_d4/strict_result", - "canonical_conversion_non_dyadic_d5/rounded_array", - "canonical_conversion_non_dyadic_d5/rounded_result", - "canonical_conversion_non_dyadic_d5/strict_array", - "canonical_conversion_non_dyadic_d5/strict_result", - "canonical_conversion_overflow_midpoint_d2/rounded_array", - "canonical_conversion_overflow_midpoint_d2/rounded_result", - "canonical_conversion_overflow_midpoint_d2/strict_array", - "canonical_conversion_overflow_midpoint_d2/strict_result", - "canonical_conversion_overflow_midpoint_d3/rounded_array", - "canonical_conversion_overflow_midpoint_d3/rounded_result", - "canonical_conversion_overflow_midpoint_d3/strict_array", - "canonical_conversion_overflow_midpoint_d3/strict_result", - "canonical_conversion_overflow_midpoint_d4/rounded_array", - "canonical_conversion_overflow_midpoint_d4/rounded_result", - "canonical_conversion_overflow_midpoint_d4/strict_array", - "canonical_conversion_overflow_midpoint_d4/strict_result", - "canonical_conversion_overflow_midpoint_d5/rounded_array", - "canonical_conversion_overflow_midpoint_d5/rounded_result", - "canonical_conversion_overflow_midpoint_d5/strict_array", - "canonical_conversion_overflow_midpoint_d5/strict_result", - "canonical_conversion_wide1024_d2/rounded_array", - "canonical_conversion_wide1024_d2/rounded_result", - "canonical_conversion_wide1024_d2/strict_array", - "canonical_conversion_wide1024_d2/strict_result", - "canonical_conversion_wide1024_d3/rounded_array", - "canonical_conversion_wide1024_d3/rounded_result", - "canonical_conversion_wide1024_d3/strict_array", - "canonical_conversion_wide1024_d3/strict_result", - "canonical_conversion_wide1024_d4/rounded_array", - "canonical_conversion_wide1024_d4/rounded_result", - "canonical_conversion_wide1024_d4/strict_array", - "canonical_conversion_wide1024_d4/strict_result", - "canonical_conversion_wide1024_d5/rounded_array", - "canonical_conversion_wide1024_d5/rounded_result", - "canonical_conversion_wide1024_d5/strict_array", - "canonical_conversion_wide1024_d5/strict_result", - "canonical_conversion_wide256_d2/rounded_array", - "canonical_conversion_wide256_d2/rounded_result", - "canonical_conversion_wide256_d2/strict_array", - "canonical_conversion_wide256_d2/strict_result", - "canonical_conversion_wide256_d3/rounded_array", - "canonical_conversion_wide256_d3/rounded_result", - "canonical_conversion_wide256_d3/strict_array", - "canonical_conversion_wide256_d3/strict_result", - "canonical_conversion_wide256_d4/rounded_array", - "canonical_conversion_wide256_d4/rounded_result", - "canonical_conversion_wide256_d4/strict_array", - "canonical_conversion_wide256_d4/strict_result", - "canonical_conversion_wide256_d5/rounded_array", - "canonical_conversion_wide256_d5/rounded_result", - "canonical_conversion_wide256_d5/strict_array", - "canonical_conversion_wide256_d5/strict_result", - "d16/faer_lu_solve", "d16/la_stack_det", "d16/la_stack_det_from_ldlt", "d16/la_stack_det_from_lu", @@ -371,7 +285,6 @@ required = [ "d16/la_stack_ldlt", "d16/la_stack_ldlt_solve", "d16/la_stack_lu", - "d16/la_stack_lu_solve", "d16/la_stack_norm2", "d16/la_stack_norm2_scenario_descending", "d16/la_stack_norm2_scenario_repeated_scale", @@ -380,8 +293,6 @@ required = [ "d16/la_stack_norm2_sq", "d16/la_stack_solve_from_ldlt", "d16/la_stack_solve_from_lu", - "d16/nalgebra_lu_solve", - "d2/faer_lu_solve", "d2/la_stack_det", "d2/la_stack_det_from_ldlt", "d2/la_stack_det_from_lu", @@ -391,7 +302,6 @@ required = [ "d2/la_stack_ldlt", "d2/la_stack_ldlt_solve", "d2/la_stack_lu", - "d2/la_stack_lu_solve", "d2/la_stack_norm2", "d2/la_stack_norm2_scenario_descending", "d2/la_stack_norm2_scenario_repeated_scale", @@ -400,8 +310,6 @@ required = [ "d2/la_stack_norm2_sq", "d2/la_stack_solve_from_ldlt", "d2/la_stack_solve_from_lu", - "d2/nalgebra_lu_solve", - "d3/faer_lu_solve", "d3/la_stack_det", "d3/la_stack_det_from_ldlt", "d3/la_stack_det_from_lu", @@ -411,7 +319,6 @@ required = [ "d3/la_stack_ldlt", "d3/la_stack_ldlt_solve", "d3/la_stack_lu", - "d3/la_stack_lu_solve", "d3/la_stack_norm2", "d3/la_stack_norm2_scenario_descending", "d3/la_stack_norm2_scenario_repeated_scale", @@ -420,8 +327,6 @@ required = [ "d3/la_stack_norm2_sq", "d3/la_stack_solve_from_ldlt", "d3/la_stack_solve_from_lu", - "d3/nalgebra_lu_solve", - "d32/faer_lu_solve", "d32/la_stack_det", "d32/la_stack_det_from_ldlt", "d32/la_stack_det_from_lu", @@ -431,7 +336,6 @@ required = [ "d32/la_stack_ldlt", "d32/la_stack_ldlt_solve", "d32/la_stack_lu", - "d32/la_stack_lu_solve", "d32/la_stack_norm2", "d32/la_stack_norm2_scenario_descending", "d32/la_stack_norm2_scenario_repeated_scale", @@ -440,8 +344,6 @@ required = [ "d32/la_stack_norm2_sq", "d32/la_stack_solve_from_ldlt", "d32/la_stack_solve_from_lu", - "d32/nalgebra_lu_solve", - "d4/faer_lu_solve", "d4/la_stack_det", "d4/la_stack_det_from_ldlt", "d4/la_stack_det_from_lu", @@ -451,7 +353,6 @@ required = [ "d4/la_stack_ldlt", "d4/la_stack_ldlt_solve", "d4/la_stack_lu", - "d4/la_stack_lu_solve", "d4/la_stack_norm2", "d4/la_stack_norm2_scenario_descending", "d4/la_stack_norm2_scenario_repeated_scale", @@ -460,8 +361,6 @@ required = [ "d4/la_stack_norm2_sq", "d4/la_stack_solve_from_ldlt", "d4/la_stack_solve_from_lu", - "d4/nalgebra_lu_solve", - "d5/faer_lu_solve", "d5/la_stack_det", "d5/la_stack_det_from_ldlt", "d5/la_stack_det_from_lu", @@ -471,7 +370,6 @@ required = [ "d5/la_stack_ldlt", "d5/la_stack_ldlt_solve", "d5/la_stack_lu", - "d5/la_stack_lu_solve", "d5/la_stack_norm2", "d5/la_stack_norm2_scenario_descending", "d5/la_stack_norm2_scenario_repeated_scale", @@ -480,8 +378,6 @@ required = [ "d5/la_stack_norm2_sq", "d5/la_stack_solve_from_ldlt", "d5/la_stack_solve_from_lu", - "d5/nalgebra_lu_solve", - "d64/faer_lu_solve", "d64/la_stack_det", "d64/la_stack_det_from_ldlt", "d64/la_stack_det_from_lu", @@ -491,7 +387,6 @@ required = [ "d64/la_stack_ldlt", "d64/la_stack_ldlt_solve", "d64/la_stack_lu", - "d64/la_stack_lu_solve", "d64/la_stack_norm2", "d64/la_stack_norm2_scenario_descending", "d64/la_stack_norm2_scenario_repeated_scale", @@ -500,8 +395,6 @@ required = [ "d64/la_stack_norm2_sq", "d64/la_stack_solve_from_ldlt", "d64/la_stack_solve_from_lu", - "d64/nalgebra_lu_solve", - "d8/faer_lu_solve", "d8/la_stack_det", "d8/la_stack_det_from_ldlt", "d8/la_stack_det_from_ldlt_balanced_range", @@ -516,7 +409,6 @@ required = [ "d8/la_stack_lu", "d8/la_stack_lu_ill_conditioned", "d8/la_stack_lu_pivoting", - "d8/la_stack_lu_solve", "d8/la_stack_norm2", "d8/la_stack_norm2_scenario_descending", "d8/la_stack_norm2_scenario_repeated_scale", @@ -525,88 +417,6 @@ required = [ "d8/la_stack_norm2_sq", "d8/la_stack_solve_from_ldlt", "d8/la_stack_solve_from_lu", - "d8/nalgebra_lu_solve", - "exact_d2/det", - "exact_d2/det_direct", - "exact_d2/det_direct_with_errbound", - "exact_d2/det_errbound", - "exact_d2/det_exact", - "exact_d2/det_exact_f64_result", - "exact_d2/det_exact_rounded_f64", - "exact_d2/det_sign_exact", - "exact_d2/solve_exact", - "exact_d2/solve_exact_f64_result", - "exact_d2/solve_exact_rounded_f64", - "exact_d3/det", - "exact_d3/det_direct", - "exact_d3/det_direct_with_errbound", - "exact_d3/det_errbound", - "exact_d3/det_exact", - "exact_d3/det_exact_f64_result", - "exact_d3/det_exact_rounded_f64", - "exact_d3/det_sign_exact", - "exact_d3/solve_exact", - "exact_d3/solve_exact_f64_result", - "exact_d3/solve_exact_rounded_f64", - "exact_d4/det", - "exact_d4/det_direct", - "exact_d4/det_direct_with_errbound", - "exact_d4/det_errbound", - "exact_d4/det_exact", - "exact_d4/det_exact_f64_result", - "exact_d4/det_exact_rounded_f64", - "exact_d4/det_sign_exact", - "exact_d4/solve_exact", - "exact_d4/solve_exact_f64_result", - "exact_d4/solve_exact_rounded_f64", - "exact_d5/det", - "exact_d5/det_exact", - "exact_d5/det_exact_f64_result", - "exact_d5/det_exact_rounded_f64", - "exact_d5/det_sign_exact", - "exact_d5/solve_exact", - "exact_d5/solve_exact_f64_result", - "exact_d5/solve_exact_rounded_f64", - "exact_hilbert_4x4/det_exact", - "exact_hilbert_4x4/det_sign_exact", - "exact_hilbert_4x4/solve_exact", - "exact_hilbert_4x4/solve_exact_f64_result", - "exact_hilbert_4x4/solve_exact_rounded_f64", - "exact_hilbert_5x5/det_exact", - "exact_hilbert_5x5/det_sign_exact", - "exact_hilbert_5x5/solve_exact", - "exact_hilbert_5x5/solve_exact_f64_result", - "exact_hilbert_5x5/solve_exact_rounded_f64", - "exact_large_entries_3x3/det_exact", - "exact_large_entries_3x3/det_sign_exact", - "exact_large_entries_3x3/solve_exact", - "exact_large_entries_3x3/solve_exact_f64_result", - "exact_large_entries_3x3/solve_exact_rounded_f64", - "exact_near_singular_3x3/det_exact", - "exact_near_singular_3x3/det_sign_exact", - "exact_near_singular_3x3/solve_exact", - "exact_near_singular_3x3/solve_exact_f64_result", - "exact_near_singular_3x3/solve_exact_rounded_f64", - "exact_random_corpus_d2/det_exact", - "exact_random_corpus_d2/det_sign_exact", - "exact_random_corpus_d2/solve_exact", - "exact_random_corpus_d2/solve_exact_f64_result", - "exact_random_corpus_d2/solve_exact_rounded_f64", - "exact_random_corpus_d3/det_exact", - "exact_random_corpus_d3/det_sign_exact", - "exact_random_corpus_d3/solve_exact", - "exact_random_corpus_d3/solve_exact_f64_result", - "exact_random_corpus_d3/solve_exact_rounded_f64", - "exact_random_corpus_d4/det_exact", - "exact_random_corpus_d4/det_sign_exact", - "exact_random_corpus_d4/solve_exact", - "exact_random_corpus_d4/solve_exact_f64_result", - "exact_random_corpus_d4/solve_exact_rounded_f64", - "exact_random_corpus_d5/det_exact", - "exact_random_corpus_d5/det_sign_exact", - "exact_random_corpus_d5/solve_exact", - "exact_random_corpus_d5/solve_exact_f64_result", - "exact_random_corpus_d5/solve_exact_rounded_f64", "gram/1x2/dependent/la_stack", "gram/1x2/mixed_scale/la_stack", "gram/1x2/near_dependent/la_stack", @@ -930,434 +740,43 @@ required = [ "public_d5/vector_access", "public_d5/vector_new", "public_d5/vector_zero", - "public_exact_d2/matrix_access", - "public_exact_d2/matrix_from_fn", - "public_exact_d2/matrix_from_rows", - "public_exact_d2/matrix_into_rows", - "public_exact_d2/matrix_zero", - "public_exact_d2/vector_access", - "public_exact_d2/vector_from_fn", - "public_exact_d2/vector_into_array", - "public_exact_d2/vector_new", - "public_exact_d2/vector_zero", - "public_exact_d3/matrix_access", - "public_exact_d3/matrix_from_fn", - "public_exact_d3/matrix_from_rows", - "public_exact_d3/matrix_into_rows", - "public_exact_d3/matrix_zero", - "public_exact_d3/vector_access", - "public_exact_d3/vector_from_fn", - "public_exact_d3/vector_into_array", - "public_exact_d3/vector_new", - "public_exact_d3/vector_zero", - "public_exact_d4/matrix_access", - "public_exact_d4/matrix_from_fn", - "public_exact_d4/matrix_from_rows", - "public_exact_d4/matrix_into_rows", - "public_exact_d4/matrix_zero", - "public_exact_d4/vector_access", - "public_exact_d4/vector_from_fn", - "public_exact_d4/vector_into_array", - "public_exact_d4/vector_new", - "public_exact_d4/vector_zero", - "public_exact_d5/matrix_access", - "public_exact_d5/matrix_from_fn", - "public_exact_d5/matrix_from_rows", - "public_exact_d5/matrix_into_rows", - "public_exact_d5/matrix_zero", - "public_exact_d5/vector_access", - "public_exact_d5/vector_from_fn", - "public_exact_d5/vector_into_array", - "public_exact_d5/vector_new", - "public_exact_d5/vector_zero", - "public_exact_scalar/rounded", - "public_exact_scalar/strict", "public_scalar/interval_access", "public_scalar/interval_negate", "public_scalar/interval_new", "public_scalar/interval_point", "public_scalar/interval_subtraction", "public_scalar/tolerance", - "rational_input_d2/det_row_cleared_bareiss", - "rational_input_d2/det_sign_row_cleared_bareiss", - "rational_input_d2/solve_row_cleared_bareiss", - "rational_input_d3/det_row_cleared_bareiss", - "rational_input_d3/det_sign_row_cleared_bareiss", - "rational_input_d3/solve_row_cleared_bareiss", - "rational_input_d4/det_row_cleared_bareiss", - "rational_input_d4/det_sign_row_cleared_bareiss", - "rational_input_d4/solve_row_cleared_bareiss", - "rational_input_d5/det_row_cleared_bareiss", - "rational_input_d5/det_sign_row_cleared_bareiss", - "rational_input_d5/solve_row_cleared_bareiss", - "rational_input_d6/det_row_cleared_bareiss", - "rational_input_d6/det_sign_row_cleared_bareiss", - "rational_input_d6/solve_row_cleared_bareiss", - "rational_input_d7/det_row_cleared_bareiss", - "rational_input_d7/det_sign_row_cleared_bareiss", - "rational_input_d7/solve_row_cleared_bareiss", - "rational_input_d8/det_row_cleared_bareiss", - "rational_input_d8/det_sign_row_cleared_bareiss", - "rational_input_d8/solve_row_cleared_bareiss", - "rational_input_wide1024_d2/det_row_cleared_bareiss", - "rational_input_wide1024_d2/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d2/solve_row_cleared_bareiss", - "rational_input_wide1024_d3/det_row_cleared_bareiss", - "rational_input_wide1024_d3/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d3/solve_row_cleared_bareiss", - "rational_input_wide1024_d4/det_row_cleared_bareiss", - "rational_input_wide1024_d4/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d4/solve_row_cleared_bareiss", - "rational_input_wide1024_d5/det_row_cleared_bareiss", - "rational_input_wide1024_d5/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d5/solve_row_cleared_bareiss", - "rational_input_wide1024_d6/det_row_cleared_bareiss", - "rational_input_wide1024_d6/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d6/solve_row_cleared_bareiss", - "rational_input_wide1024_d7/det_row_cleared_bareiss", - "rational_input_wide1024_d7/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d7/solve_row_cleared_bareiss", - "rational_input_wide1024_d8/det_row_cleared_bareiss", - "rational_input_wide1024_d8/det_sign_row_cleared_bareiss", - "rational_input_wide1024_d8/solve_row_cleared_bareiss", - "rational_input_wide256_d2/det_row_cleared_bareiss", - "rational_input_wide256_d2/det_sign_row_cleared_bareiss", - "rational_input_wide256_d2/solve_row_cleared_bareiss", - "rational_input_wide256_d3/det_row_cleared_bareiss", - "rational_input_wide256_d3/det_sign_row_cleared_bareiss", - "rational_input_wide256_d3/solve_row_cleared_bareiss", - "rational_input_wide256_d4/det_row_cleared_bareiss", - "rational_input_wide256_d4/det_sign_row_cleared_bareiss", - "rational_input_wide256_d4/solve_row_cleared_bareiss", - "rational_input_wide256_d5/det_row_cleared_bareiss", - "rational_input_wide256_d5/det_sign_row_cleared_bareiss", - "rational_input_wide256_d5/solve_row_cleared_bareiss", - "rational_input_wide256_d6/det_row_cleared_bareiss", - "rational_input_wide256_d6/det_sign_row_cleared_bareiss", - "rational_input_wide256_d6/solve_row_cleared_bareiss", - "rational_input_wide256_d7/det_row_cleared_bareiss", - "rational_input_wide256_d7/det_sign_row_cleared_bareiss", - "rational_input_wide256_d7/solve_row_cleared_bareiss", - "rational_input_wide256_d8/det_row_cleared_bareiss", - "rational_input_wide256_d8/det_sign_row_cleared_bareiss", - "rational_input_wide256_d8/solve_row_cleared_bareiss", - "rational_solve/dense/dyadic_d2/adapter", - "rational_solve/dense/dyadic_d2/construct", - "rational_solve/dense/dyadic_d2/prepared", - "rational_solve/dense/dyadic_d3/adapter", - "rational_solve/dense/dyadic_d3/construct", - "rational_solve/dense/dyadic_d3/prepared", - "rational_solve/dense/dyadic_d4/adapter", - "rational_solve/dense/dyadic_d4/construct", - "rational_solve/dense/dyadic_d4/prepared", - "rational_solve/dense/dyadic_d5/adapter", - "rational_solve/dense/dyadic_d5/construct", - "rational_solve/dense/dyadic_d5/prepared", - "rational_solve/dense/dyadic_d6/adapter", - "rational_solve/dense/dyadic_d6/construct", - "rational_solve/dense/dyadic_d6/prepared", - "rational_solve/dense/dyadic_d7/adapter", - "rational_solve/dense/dyadic_d7/construct", - "rational_solve/dense/dyadic_d7/prepared", - "rational_solve/dense/dyadic_d8/adapter", - "rational_solve/dense/dyadic_d8/construct", - "rational_solve/dense/dyadic_d8/prepared", - "rational_solve/dense/rational_d2/adapter", - "rational_solve/dense/rational_d2/construct", - "rational_solve/dense/rational_d2/prepared", - "rational_solve/dense/rational_d3/adapter", - "rational_solve/dense/rational_d3/construct", - "rational_solve/dense/rational_d3/prepared", - "rational_solve/dense/rational_d4/adapter", - "rational_solve/dense/rational_d4/construct", - "rational_solve/dense/rational_d4/prepared", - "rational_solve/dense/rational_d5/adapter", - "rational_solve/dense/rational_d5/construct", - "rational_solve/dense/rational_d5/prepared", - "rational_solve/dense/rational_d6/adapter", - "rational_solve/dense/rational_d6/construct", - "rational_solve/dense/rational_d6/prepared", - "rational_solve/dense/rational_d7/adapter", - "rational_solve/dense/rational_d7/construct", - "rational_solve/dense/rational_d7/prepared", - "rational_solve/dense/rational_d8/adapter", - "rational_solve/dense/rational_d8/construct", - "rational_solve/dense/rational_d8/prepared", - "rational_solve/diagonal/dyadic_d2/adapter", - "rational_solve/diagonal/dyadic_d2/construct", - "rational_solve/diagonal/dyadic_d2/prepared", - "rational_solve/diagonal/dyadic_d3/adapter", - "rational_solve/diagonal/dyadic_d3/construct", - "rational_solve/diagonal/dyadic_d3/prepared", - "rational_solve/diagonal/dyadic_d4/adapter", - "rational_solve/diagonal/dyadic_d4/construct", - "rational_solve/diagonal/dyadic_d4/prepared", - "rational_solve/diagonal/dyadic_d5/adapter", - "rational_solve/diagonal/dyadic_d5/construct", - "rational_solve/diagonal/dyadic_d5/prepared", - "rational_solve/diagonal/dyadic_d6/adapter", - "rational_solve/diagonal/dyadic_d6/construct", - "rational_solve/diagonal/dyadic_d6/prepared", - "rational_solve/diagonal/dyadic_d7/adapter", - "rational_solve/diagonal/dyadic_d7/construct", - "rational_solve/diagonal/dyadic_d7/prepared", - "rational_solve/diagonal/dyadic_d8/adapter", - "rational_solve/diagonal/dyadic_d8/construct", - "rational_solve/diagonal/dyadic_d8/prepared", - "rational_solve/diagonal/rational_d2/adapter", - "rational_solve/diagonal/rational_d2/construct", - "rational_solve/diagonal/rational_d2/prepared", - "rational_solve/diagonal/rational_d3/adapter", - "rational_solve/diagonal/rational_d3/construct", - "rational_solve/diagonal/rational_d3/prepared", - "rational_solve/diagonal/rational_d4/adapter", - "rational_solve/diagonal/rational_d4/construct", - "rational_solve/diagonal/rational_d4/prepared", - "rational_solve/diagonal/rational_d5/adapter", - "rational_solve/diagonal/rational_d5/construct", - "rational_solve/diagonal/rational_d5/prepared", - "rational_solve/diagonal/rational_d6/adapter", - "rational_solve/diagonal/rational_d6/construct", - "rational_solve/diagonal/rational_d6/prepared", - "rational_solve/diagonal/rational_d7/adapter", - "rational_solve/diagonal/rational_d7/construct", - "rational_solve/diagonal/rational_d7/prepared", - "rational_solve/diagonal/rational_d8/adapter", - "rational_solve/diagonal/rational_d8/construct", - "rational_solve/diagonal/rational_d8/prepared", - "rational_solve/lower/dyadic_d2/adapter", - "rational_solve/lower/dyadic_d2/construct", - "rational_solve/lower/dyadic_d2/prepared", - "rational_solve/lower/dyadic_d3/adapter", - "rational_solve/lower/dyadic_d3/construct", - "rational_solve/lower/dyadic_d3/prepared", - "rational_solve/lower/dyadic_d4/adapter", - "rational_solve/lower/dyadic_d4/construct", - "rational_solve/lower/dyadic_d4/prepared", - "rational_solve/lower/dyadic_d5/adapter", - "rational_solve/lower/dyadic_d5/construct", - "rational_solve/lower/dyadic_d5/prepared", - "rational_solve/lower/dyadic_d6/adapter", - "rational_solve/lower/dyadic_d6/construct", - "rational_solve/lower/dyadic_d6/prepared", - "rational_solve/lower/dyadic_d7/adapter", - "rational_solve/lower/dyadic_d7/construct", - "rational_solve/lower/dyadic_d7/prepared", - "rational_solve/lower/dyadic_d8/adapter", - "rational_solve/lower/dyadic_d8/construct", - "rational_solve/lower/dyadic_d8/prepared", - "rational_solve/lower/rational_d2/adapter", - "rational_solve/lower/rational_d2/construct", - "rational_solve/lower/rational_d2/prepared", - "rational_solve/lower/rational_d3/adapter", - "rational_solve/lower/rational_d3/construct", - "rational_solve/lower/rational_d3/prepared", - "rational_solve/lower/rational_d4/adapter", - "rational_solve/lower/rational_d4/construct", - "rational_solve/lower/rational_d4/prepared", - "rational_solve/lower/rational_d5/adapter", - "rational_solve/lower/rational_d5/construct", - "rational_solve/lower/rational_d5/prepared", - "rational_solve/lower/rational_d6/adapter", - "rational_solve/lower/rational_d6/construct", - "rational_solve/lower/rational_d6/prepared", - "rational_solve/lower/rational_d7/adapter", - "rational_solve/lower/rational_d7/construct", - "rational_solve/lower/rational_d7/prepared", - "rational_solve/lower/rational_d8/adapter", - "rational_solve/lower/rational_d8/construct", - "rational_solve/lower/rational_d8/prepared", - "rational_solve/permuted_lower/dyadic_d2/adapter", - "rational_solve/permuted_lower/dyadic_d2/construct", - "rational_solve/permuted_lower/dyadic_d2/prepared", - "rational_solve/permuted_lower/dyadic_d3/adapter", - "rational_solve/permuted_lower/dyadic_d3/construct", - "rational_solve/permuted_lower/dyadic_d3/prepared", - "rational_solve/permuted_lower/dyadic_d4/adapter", - "rational_solve/permuted_lower/dyadic_d4/construct", - "rational_solve/permuted_lower/dyadic_d4/prepared", - "rational_solve/permuted_lower/dyadic_d5/adapter", - "rational_solve/permuted_lower/dyadic_d5/construct", - "rational_solve/permuted_lower/dyadic_d5/prepared", - "rational_solve/permuted_lower/dyadic_d6/adapter", - "rational_solve/permuted_lower/dyadic_d6/construct", - "rational_solve/permuted_lower/dyadic_d6/prepared", - "rational_solve/permuted_lower/dyadic_d7/adapter", - "rational_solve/permuted_lower/dyadic_d7/construct", - "rational_solve/permuted_lower/dyadic_d7/prepared", - "rational_solve/permuted_lower/dyadic_d8/adapter", - "rational_solve/permuted_lower/dyadic_d8/construct", - "rational_solve/permuted_lower/dyadic_d8/prepared", - "rational_solve/permuted_lower/rational_d2/adapter", - "rational_solve/permuted_lower/rational_d2/construct", - "rational_solve/permuted_lower/rational_d2/prepared", - "rational_solve/permuted_lower/rational_d3/adapter", - "rational_solve/permuted_lower/rational_d3/construct", - "rational_solve/permuted_lower/rational_d3/prepared", - "rational_solve/permuted_lower/rational_d4/adapter", - "rational_solve/permuted_lower/rational_d4/construct", - "rational_solve/permuted_lower/rational_d4/prepared", - "rational_solve/permuted_lower/rational_d5/adapter", - "rational_solve/permuted_lower/rational_d5/construct", - "rational_solve/permuted_lower/rational_d5/prepared", - "rational_solve/permuted_lower/rational_d6/adapter", - "rational_solve/permuted_lower/rational_d6/construct", - "rational_solve/permuted_lower/rational_d6/prepared", - "rational_solve/permuted_lower/rational_d7/adapter", - "rational_solve/permuted_lower/rational_d7/construct", - "rational_solve/permuted_lower/rational_d7/prepared", - "rational_solve/permuted_lower/rational_d8/adapter", - "rational_solve/permuted_lower/rational_d8/construct", - "rational_solve/permuted_lower/rational_d8/prepared", - "rational_solve/permuted_upper/dyadic_d2/adapter", - "rational_solve/permuted_upper/dyadic_d2/construct", - "rational_solve/permuted_upper/dyadic_d2/prepared", - "rational_solve/permuted_upper/dyadic_d3/adapter", - "rational_solve/permuted_upper/dyadic_d3/construct", - "rational_solve/permuted_upper/dyadic_d3/prepared", - "rational_solve/permuted_upper/dyadic_d4/adapter", - "rational_solve/permuted_upper/dyadic_d4/construct", - "rational_solve/permuted_upper/dyadic_d4/prepared", - "rational_solve/permuted_upper/dyadic_d5/adapter", - "rational_solve/permuted_upper/dyadic_d5/construct", - "rational_solve/permuted_upper/dyadic_d5/prepared", - "rational_solve/permuted_upper/dyadic_d6/adapter", - "rational_solve/permuted_upper/dyadic_d6/construct", - "rational_solve/permuted_upper/dyadic_d6/prepared", - "rational_solve/permuted_upper/dyadic_d7/adapter", - "rational_solve/permuted_upper/dyadic_d7/construct", - "rational_solve/permuted_upper/dyadic_d7/prepared", - "rational_solve/permuted_upper/dyadic_d8/adapter", - "rational_solve/permuted_upper/dyadic_d8/construct", - "rational_solve/permuted_upper/dyadic_d8/prepared", - "rational_solve/permuted_upper/rational_d2/adapter", - "rational_solve/permuted_upper/rational_d2/construct", - "rational_solve/permuted_upper/rational_d2/prepared", - "rational_solve/permuted_upper/rational_d3/adapter", - "rational_solve/permuted_upper/rational_d3/construct", - "rational_solve/permuted_upper/rational_d3/prepared", - "rational_solve/permuted_upper/rational_d4/adapter", - "rational_solve/permuted_upper/rational_d4/construct", - "rational_solve/permuted_upper/rational_d4/prepared", - "rational_solve/permuted_upper/rational_d5/adapter", - "rational_solve/permuted_upper/rational_d5/construct", - "rational_solve/permuted_upper/rational_d5/prepared", - "rational_solve/permuted_upper/rational_d6/adapter", - "rational_solve/permuted_upper/rational_d6/construct", - "rational_solve/permuted_upper/rational_d6/prepared", - "rational_solve/permuted_upper/rational_d7/adapter", - "rational_solve/permuted_upper/rational_d7/construct", - "rational_solve/permuted_upper/rational_d7/prepared", - "rational_solve/permuted_upper/rational_d8/adapter", - "rational_solve/permuted_upper/rational_d8/construct", - "rational_solve/permuted_upper/rational_d8/prepared", - "rational_solve/sparse/dyadic_d2/adapter", - "rational_solve/sparse/dyadic_d2/construct", - "rational_solve/sparse/dyadic_d2/prepared", - "rational_solve/sparse/dyadic_d3/adapter", - "rational_solve/sparse/dyadic_d3/construct", - "rational_solve/sparse/dyadic_d3/prepared", - "rational_solve/sparse/dyadic_d4/adapter", - "rational_solve/sparse/dyadic_d4/construct", - "rational_solve/sparse/dyadic_d4/prepared", - "rational_solve/sparse/dyadic_d5/adapter", - "rational_solve/sparse/dyadic_d5/construct", - "rational_solve/sparse/dyadic_d5/prepared", - "rational_solve/sparse/dyadic_d6/adapter", - "rational_solve/sparse/dyadic_d6/construct", - "rational_solve/sparse/dyadic_d6/prepared", - "rational_solve/sparse/dyadic_d7/adapter", - "rational_solve/sparse/dyadic_d7/construct", - "rational_solve/sparse/dyadic_d7/prepared", - "rational_solve/sparse/dyadic_d8/adapter", - "rational_solve/sparse/dyadic_d8/construct", - "rational_solve/sparse/dyadic_d8/prepared", - "rational_solve/sparse/rational_d2/adapter", - "rational_solve/sparse/rational_d2/construct", - "rational_solve/sparse/rational_d2/prepared", - "rational_solve/sparse/rational_d3/adapter", - "rational_solve/sparse/rational_d3/construct", - "rational_solve/sparse/rational_d3/prepared", - "rational_solve/sparse/rational_d4/adapter", - "rational_solve/sparse/rational_d4/construct", - "rational_solve/sparse/rational_d4/prepared", - "rational_solve/sparse/rational_d5/adapter", - "rational_solve/sparse/rational_d5/construct", - "rational_solve/sparse/rational_d5/prepared", - "rational_solve/sparse/rational_d6/adapter", - "rational_solve/sparse/rational_d6/construct", - "rational_solve/sparse/rational_d6/prepared", - "rational_solve/sparse/rational_d7/adapter", - "rational_solve/sparse/rational_d7/construct", - "rational_solve/sparse/rational_d7/prepared", - "rational_solve/sparse/rational_d8/adapter", - "rational_solve/sparse/rational_d8/construct", - "rational_solve/sparse/rational_d8/prepared", - "rational_solve/upper/dyadic_d2/adapter", - "rational_solve/upper/dyadic_d2/construct", - "rational_solve/upper/dyadic_d2/prepared", - "rational_solve/upper/dyadic_d3/adapter", - "rational_solve/upper/dyadic_d3/construct", - "rational_solve/upper/dyadic_d3/prepared", - "rational_solve/upper/dyadic_d4/adapter", - "rational_solve/upper/dyadic_d4/construct", - "rational_solve/upper/dyadic_d4/prepared", - "rational_solve/upper/dyadic_d5/adapter", - "rational_solve/upper/dyadic_d5/construct", - "rational_solve/upper/dyadic_d5/prepared", - "rational_solve/upper/dyadic_d6/adapter", - "rational_solve/upper/dyadic_d6/construct", - "rational_solve/upper/dyadic_d6/prepared", - "rational_solve/upper/dyadic_d7/adapter", - "rational_solve/upper/dyadic_d7/construct", - "rational_solve/upper/dyadic_d7/prepared", - "rational_solve/upper/dyadic_d8/adapter", - "rational_solve/upper/dyadic_d8/construct", - "rational_solve/upper/dyadic_d8/prepared", - "rational_solve/upper/rational_d2/adapter", - "rational_solve/upper/rational_d2/construct", - "rational_solve/upper/rational_d2/prepared", - "rational_solve/upper/rational_d3/adapter", - "rational_solve/upper/rational_d3/construct", - "rational_solve/upper/rational_d3/prepared", - "rational_solve/upper/rational_d4/adapter", - "rational_solve/upper/rational_d4/construct", - "rational_solve/upper/rational_d4/prepared", - "rational_solve/upper/rational_d5/adapter", - "rational_solve/upper/rational_d5/construct", - "rational_solve/upper/rational_d5/prepared", - "rational_solve/upper/rational_d6/adapter", - "rational_solve/upper/rational_d6/construct", - "rational_solve/upper/rational_d6/prepared", - "rational_solve/upper/rational_d7/adapter", - "rational_solve/upper/rational_d7/construct", - "rational_solve/upper/rational_d7/prepared", - "rational_solve/upper/rational_d8/adapter", - "rational_solve/upper/rational_d8/construct", - "rational_solve/upper/rational_d8/prepared", ] -[[phases.baseline.discovery.suites]] -name = "public-default" -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-public-default", "list" ] -timeout = 14400 - -[[phases.baseline.discovery.suites]] -name = "public-exact" -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-public-exact", "list" ] +[[suites.measurement.phases.baseline.discovery.suites]] +name = "core-default" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "core-default", + "list", +] timeout = 14400 -[[phases.baseline.discovery.suites]] -name = "peers" -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-peers", "list" ] -timeout = 14400 +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "core-default" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-current" ] -[phases.current] -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "current-all" ] -gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-public" ] -environment = { CARGO_TARGET_DIR = "target", CRITERION_HOME = "target/criterion", RUSTUP_TOOLCHAIN = "1.99.1", RUSTFLAGS = "--cap-lints=warn", CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn", RUSTC = "rustc", RUSTC_WRAPPER = "", RUSTC_WORKSPACE_WRAPPER = "" } +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" -[phases.current.discovery] +[suites.measurement.phases.current.discovery] required = [ "angle_d3/dense/borrowed_slice", "angle_d3/dense/construct_then_angle", @@ -1527,189 +946,6 @@ required = [ "angle_d6/subnormal_axes/borrowed_slice", "angle_d6/subnormal_axes/construct_then_angle", "angle_d6/subnormal_axes/prepared_vector", - "canonical_conversion_below_overflow_d2/rounded_array", - "canonical_conversion_below_overflow_d2/rounded_result", - "canonical_conversion_below_overflow_d2/strict_array", - "canonical_conversion_below_overflow_d2/strict_result", - "canonical_conversion_below_overflow_d3/rounded_array", - "canonical_conversion_below_overflow_d3/rounded_result", - "canonical_conversion_below_overflow_d3/strict_array", - "canonical_conversion_below_overflow_d3/strict_result", - "canonical_conversion_below_overflow_d4/rounded_array", - "canonical_conversion_below_overflow_d4/rounded_result", - "canonical_conversion_below_overflow_d4/strict_array", - "canonical_conversion_below_overflow_d4/strict_result", - "canonical_conversion_below_overflow_d5/rounded_array", - "canonical_conversion_below_overflow_d5/rounded_result", - "canonical_conversion_below_overflow_d5/strict_array", - "canonical_conversion_below_overflow_d5/strict_result", - "canonical_conversion_dyadic_d2/rounded_array", - "canonical_conversion_dyadic_d2/rounded_result", - "canonical_conversion_dyadic_d2/strict_array", - "canonical_conversion_dyadic_d2/strict_result", - "canonical_conversion_dyadic_d3/rounded_array", - "canonical_conversion_dyadic_d3/rounded_result", - "canonical_conversion_dyadic_d3/strict_array", - "canonical_conversion_dyadic_d3/strict_result", - "canonical_conversion_dyadic_d4/rounded_array", - "canonical_conversion_dyadic_d4/rounded_result", - "canonical_conversion_dyadic_d4/strict_array", - "canonical_conversion_dyadic_d4/strict_result", - "canonical_conversion_dyadic_d5/rounded_array", - "canonical_conversion_dyadic_d5/rounded_result", - "canonical_conversion_dyadic_d5/strict_array", - "canonical_conversion_dyadic_d5/strict_result", - "canonical_conversion_min_subnormal_d2/rounded_array", - "canonical_conversion_min_subnormal_d2/rounded_result", - "canonical_conversion_min_subnormal_d2/strict_array", - "canonical_conversion_min_subnormal_d2/strict_result", - "canonical_conversion_min_subnormal_d3/rounded_array", - "canonical_conversion_min_subnormal_d3/rounded_result", - "canonical_conversion_min_subnormal_d3/strict_array", - "canonical_conversion_min_subnormal_d3/strict_result", - "canonical_conversion_min_subnormal_d4/rounded_array", - "canonical_conversion_min_subnormal_d4/rounded_result", - "canonical_conversion_min_subnormal_d4/strict_array", - "canonical_conversion_min_subnormal_d4/strict_result", - "canonical_conversion_min_subnormal_d5/rounded_array", - "canonical_conversion_min_subnormal_d5/rounded_result", - "canonical_conversion_min_subnormal_d5/strict_array", - "canonical_conversion_min_subnormal_d5/strict_result", - "canonical_conversion_negative_underflow_d2/rounded_array", - "canonical_conversion_negative_underflow_d2/rounded_result", - "canonical_conversion_negative_underflow_d2/strict_array", - "canonical_conversion_negative_underflow_d2/strict_result", - "canonical_conversion_negative_underflow_d3/rounded_array", - "canonical_conversion_negative_underflow_d3/rounded_result", - "canonical_conversion_negative_underflow_d3/strict_array", - "canonical_conversion_negative_underflow_d3/strict_result", - "canonical_conversion_negative_underflow_d4/rounded_array", - "canonical_conversion_negative_underflow_d4/rounded_result", - "canonical_conversion_negative_underflow_d4/strict_array", - "canonical_conversion_negative_underflow_d4/strict_result", - "canonical_conversion_negative_underflow_d5/rounded_array", - "canonical_conversion_negative_underflow_d5/rounded_result", - "canonical_conversion_negative_underflow_d5/strict_array", - "canonical_conversion_negative_underflow_d5/strict_result", - "canonical_conversion_non_dyadic_d2/rounded_array", - "canonical_conversion_non_dyadic_d2/rounded_result", - "canonical_conversion_non_dyadic_d2/strict_array", - "canonical_conversion_non_dyadic_d2/strict_result", - "canonical_conversion_non_dyadic_d3/rounded_array", - "canonical_conversion_non_dyadic_d3/rounded_result", - "canonical_conversion_non_dyadic_d3/strict_array", - "canonical_conversion_non_dyadic_d3/strict_result", - "canonical_conversion_non_dyadic_d4/rounded_array", - "canonical_conversion_non_dyadic_d4/rounded_result", - "canonical_conversion_non_dyadic_d4/strict_array", - "canonical_conversion_non_dyadic_d4/strict_result", - "canonical_conversion_non_dyadic_d5/rounded_array", - "canonical_conversion_non_dyadic_d5/rounded_result", - "canonical_conversion_non_dyadic_d5/strict_array", - "canonical_conversion_non_dyadic_d5/strict_result", - "canonical_conversion_overflow_midpoint_d2/rounded_array", - "canonical_conversion_overflow_midpoint_d2/rounded_result", - "canonical_conversion_overflow_midpoint_d2/strict_array", - "canonical_conversion_overflow_midpoint_d2/strict_result", - "canonical_conversion_overflow_midpoint_d3/rounded_array", - "canonical_conversion_overflow_midpoint_d3/rounded_result", - "canonical_conversion_overflow_midpoint_d3/strict_array", - "canonical_conversion_overflow_midpoint_d3/strict_result", - "canonical_conversion_overflow_midpoint_d4/rounded_array", - "canonical_conversion_overflow_midpoint_d4/rounded_result", - "canonical_conversion_overflow_midpoint_d4/strict_array", - "canonical_conversion_overflow_midpoint_d4/strict_result", - "canonical_conversion_overflow_midpoint_d5/rounded_array", - "canonical_conversion_overflow_midpoint_d5/rounded_result", - "canonical_conversion_overflow_midpoint_d5/strict_array", - "canonical_conversion_overflow_midpoint_d5/strict_result", - "canonical_conversion_wide1024_d2/rounded_array", - "canonical_conversion_wide1024_d2/rounded_result", - "canonical_conversion_wide1024_d2/strict_array", - "canonical_conversion_wide1024_d2/strict_result", - "canonical_conversion_wide1024_d3/rounded_array", - "canonical_conversion_wide1024_d3/rounded_result", - "canonical_conversion_wide1024_d3/strict_array", - "canonical_conversion_wide1024_d3/strict_result", - "canonical_conversion_wide1024_d4/rounded_array", - "canonical_conversion_wide1024_d4/rounded_result", - "canonical_conversion_wide1024_d4/strict_array", - "canonical_conversion_wide1024_d4/strict_result", - "canonical_conversion_wide1024_d5/rounded_array", - "canonical_conversion_wide1024_d5/rounded_result", - "canonical_conversion_wide1024_d5/strict_array", - "canonical_conversion_wide1024_d5/strict_result", - "canonical_conversion_wide256_d2/rounded_array", - "canonical_conversion_wide256_d2/rounded_result", - "canonical_conversion_wide256_d2/strict_array", - "canonical_conversion_wide256_d2/strict_result", - "canonical_conversion_wide256_d3/rounded_array", - "canonical_conversion_wide256_d3/rounded_result", - "canonical_conversion_wide256_d3/strict_array", - "canonical_conversion_wide256_d3/strict_result", - "canonical_conversion_wide256_d4/rounded_array", - "canonical_conversion_wide256_d4/rounded_result", - "canonical_conversion_wide256_d4/strict_array", - "canonical_conversion_wide256_d4/strict_result", - "canonical_conversion_wide256_d5/rounded_array", - "canonical_conversion_wide256_d5/rounded_result", - "canonical_conversion_wide256_d5/strict_array", - "canonical_conversion_wide256_d5/strict_result", - "column_basis/2x2/full_rank", - "column_basis/2x2/large", - "column_basis/2x2/near_dependent", - "column_basis/2x2/rank_one", - "column_basis/2x2/small", - "column_basis/2x3/shared_face", - "column_basis/3x3/full_rank", - "column_basis/3x3/large", - "column_basis/3x3/near_dependent", - "column_basis/3x3/rank_one", - "column_basis/3x3/small", - "column_basis/3x4/shared_face", - "column_basis/4x4/full_rank", - "column_basis/4x4/large", - "column_basis/4x4/near_dependent", - "column_basis/4x4/rank_one", - "column_basis/4x4/small", - "column_basis/4x5/shared_face", - "column_basis/4x6/full_rank", - "column_basis/4x6/large", - "column_basis/4x6/near_dependent", - "column_basis/4x6/rank_one", - "column_basis/4x6/small", - "column_basis/5x2/full_rank", - "column_basis/5x2/large", - "column_basis/5x2/near_dependent", - "column_basis/5x2/rank_one", - "column_basis/5x2/small", - "column_basis/5x5/full_rank", - "column_basis/5x5/large", - "column_basis/5x5/near_dependent", - "column_basis/5x5/rank_one", - "column_basis/5x5/small", - "column_basis/5x6/shared_face", - "column_basis/5x8/full_rank", - "column_basis/5x8/large", - "column_basis/5x8/near_dependent", - "column_basis/5x8/rank_one", - "column_basis/5x8/small", - "column_basis/6x10/full_rank", - "column_basis/6x10/large", - "column_basis/6x10/near_dependent", - "column_basis/6x10/rank_one", - "column_basis/6x10/small", - "column_basis/6x7/shared_face", - "column_basis/7x12/full_rank", - "column_basis/7x12/large", - "column_basis/7x12/near_dependent", - "column_basis/7x12/rank_one", - "column_basis/7x12/small", - "column_basis/8x14/full_rank", - "column_basis/8x14/large", - "column_basis/8x14/near_dependent", - "column_basis/8x14/rank_one", - "column_basis/8x14/small", "d16/la_stack_det", "d16/la_stack_det_from_ldlt", "d16/la_stack_det_from_lu", @@ -1719,7 +955,6 @@ required = [ "d16/la_stack_ldlt", "d16/la_stack_ldlt_solve", "d16/la_stack_lu", - "d16/la_stack_lu_solve", "d16/la_stack_norm2", "d16/la_stack_norm2_scenario_descending", "d16/la_stack_norm2_scenario_repeated_scale", @@ -1737,7 +972,6 @@ required = [ "d2/la_stack_ldlt", "d2/la_stack_ldlt_solve", "d2/la_stack_lu", - "d2/la_stack_lu_solve", "d2/la_stack_norm2", "d2/la_stack_norm2_scenario_descending", "d2/la_stack_norm2_scenario_repeated_scale", @@ -1755,7 +989,6 @@ required = [ "d3/la_stack_ldlt", "d3/la_stack_ldlt_solve", "d3/la_stack_lu", - "d3/la_stack_lu_solve", "d3/la_stack_norm2", "d3/la_stack_norm2_scenario_descending", "d3/la_stack_norm2_scenario_repeated_scale", @@ -1773,7 +1006,6 @@ required = [ "d32/la_stack_ldlt", "d32/la_stack_ldlt_solve", "d32/la_stack_lu", - "d32/la_stack_lu_solve", "d32/la_stack_norm2", "d32/la_stack_norm2_scenario_descending", "d32/la_stack_norm2_scenario_repeated_scale", @@ -1791,7 +1023,6 @@ required = [ "d4/la_stack_ldlt", "d4/la_stack_ldlt_solve", "d4/la_stack_lu", - "d4/la_stack_lu_solve", "d4/la_stack_norm2", "d4/la_stack_norm2_scenario_descending", "d4/la_stack_norm2_scenario_repeated_scale", @@ -1809,7 +1040,6 @@ required = [ "d5/la_stack_ldlt", "d5/la_stack_ldlt_solve", "d5/la_stack_lu", - "d5/la_stack_lu_solve", "d5/la_stack_norm2", "d5/la_stack_norm2_scenario_descending", "d5/la_stack_norm2_scenario_repeated_scale", @@ -1827,7 +1057,6 @@ required = [ "d64/la_stack_ldlt", "d64/la_stack_ldlt_solve", "d64/la_stack_lu", - "d64/la_stack_lu_solve", "d64/la_stack_norm2", "d64/la_stack_norm2_scenario_descending", "d64/la_stack_norm2_scenario_repeated_scale", @@ -1850,7 +1079,6 @@ required = [ "d8/la_stack_lu", "d8/la_stack_lu_ill_conditioned", "d8/la_stack_lu_pivoting", - "d8/la_stack_lu_solve", "d8/la_stack_norm2", "d8/la_stack_norm2_scenario_descending", "d8/la_stack_norm2_scenario_repeated_scale", @@ -1859,87 +1087,6 @@ required = [ "d8/la_stack_norm2_sq", "d8/la_stack_solve_from_ldlt", "d8/la_stack_solve_from_lu", - "exact_d2/det", - "exact_d2/det_direct", - "exact_d2/det_direct_with_errbound", - "exact_d2/det_errbound", - "exact_d2/det_exact", - "exact_d2/det_exact_f64_result", - "exact_d2/det_exact_rounded_f64", - "exact_d2/det_sign_exact", - "exact_d2/solve_exact", - "exact_d2/solve_exact_f64_result", - "exact_d2/solve_exact_rounded_f64", - "exact_d3/det", - "exact_d3/det_direct", - "exact_d3/det_direct_with_errbound", - "exact_d3/det_errbound", - "exact_d3/det_exact", - "exact_d3/det_exact_f64_result", - "exact_d3/det_exact_rounded_f64", - "exact_d3/det_sign_exact", - "exact_d3/solve_exact", - "exact_d3/solve_exact_f64_result", - "exact_d3/solve_exact_rounded_f64", - "exact_d4/det", - "exact_d4/det_direct", - "exact_d4/det_direct_with_errbound", - "exact_d4/det_errbound", - "exact_d4/det_exact", - "exact_d4/det_exact_f64_result", - "exact_d4/det_exact_rounded_f64", - "exact_d4/det_sign_exact", - "exact_d4/solve_exact", - "exact_d4/solve_exact_f64_result", - "exact_d4/solve_exact_rounded_f64", - "exact_d5/det", - "exact_d5/det_exact", - "exact_d5/det_exact_f64_result", - "exact_d5/det_exact_rounded_f64", - "exact_d5/det_sign_exact", - "exact_d5/solve_exact", - "exact_d5/solve_exact_f64_result", - "exact_d5/solve_exact_rounded_f64", - "exact_hilbert_4x4/det_exact", - "exact_hilbert_4x4/det_sign_exact", - "exact_hilbert_4x4/solve_exact", - "exact_hilbert_4x4/solve_exact_f64_result", - "exact_hilbert_4x4/solve_exact_rounded_f64", - "exact_hilbert_5x5/det_exact", - "exact_hilbert_5x5/det_sign_exact", - "exact_hilbert_5x5/solve_exact", - "exact_hilbert_5x5/solve_exact_f64_result", - "exact_hilbert_5x5/solve_exact_rounded_f64", - "exact_large_entries_3x3/det_exact", - "exact_large_entries_3x3/det_sign_exact", - "exact_large_entries_3x3/solve_exact", - "exact_large_entries_3x3/solve_exact_f64_result", - "exact_large_entries_3x3/solve_exact_rounded_f64", - "exact_near_singular_3x3/det_exact", - "exact_near_singular_3x3/det_sign_exact", - "exact_near_singular_3x3/solve_exact", - "exact_near_singular_3x3/solve_exact_f64_result", - "exact_near_singular_3x3/solve_exact_rounded_f64", - "exact_random_corpus_d2/det_exact", - "exact_random_corpus_d2/det_sign_exact", - "exact_random_corpus_d2/solve_exact", - "exact_random_corpus_d2/solve_exact_f64_result", - "exact_random_corpus_d2/solve_exact_rounded_f64", - "exact_random_corpus_d3/det_exact", - "exact_random_corpus_d3/det_sign_exact", - "exact_random_corpus_d3/solve_exact", - "exact_random_corpus_d3/solve_exact_f64_result", - "exact_random_corpus_d3/solve_exact_rounded_f64", - "exact_random_corpus_d4/det_exact", - "exact_random_corpus_d4/det_sign_exact", - "exact_random_corpus_d4/solve_exact", - "exact_random_corpus_d4/solve_exact_f64_result", - "exact_random_corpus_d4/solve_exact_rounded_f64", - "exact_random_corpus_d5/det_exact", - "exact_random_corpus_d5/det_sign_exact", - "exact_random_corpus_d5/solve_exact", - "exact_random_corpus_d5/solve_exact_f64_result", - "exact_random_corpus_d5/solve_exact_rounded_f64", "gram/1x2/dependent/la_stack", "gram/1x2/mixed_scale/la_stack", "gram/1x2/near_dependent/la_stack", @@ -2024,11 +1171,6 @@ required = [ "interval_det_sign/d4_conclusive_lifted", "interval_det_sign/d4_inconclusive_lifted", "interval_det_sign/d7_conclusive_lifted", - "interval_division/negative", - "interval_division/positive", - "interval_division/reciprocal", - "interval_division/subnormal", - "interval_division/tiny_ratio", "interval_scalar/cancellation/add", "interval_scalar/cancellation/multiply", "interval_scalar/cancellation/square", @@ -2044,36 +1186,6 @@ required = [ "interval_scalar/zero/add", "interval_scalar/zero/multiply", "interval_scalar/zero/square", - "interval_solve_d1/dense", - "interval_solve_d1/near_singular", - "interval_solve_d1/range_exhausted", - "interval_solve_d1/singular", - "interval_solve_d1/uncertain_dense", - "interval_solve_d2/dense", - "interval_solve_d2/near_singular", - "interval_solve_d2/range_exhausted", - "interval_solve_d2/singular", - "interval_solve_d2/uncertain_dense", - "interval_solve_d3/dense", - "interval_solve_d3/near_singular", - "interval_solve_d3/range_exhausted", - "interval_solve_d3/singular", - "interval_solve_d3/uncertain_dense", - "interval_solve_d4/dense", - "interval_solve_d4/near_singular", - "interval_solve_d4/range_exhausted", - "interval_solve_d4/singular", - "interval_solve_d4/uncertain_dense", - "interval_solve_d5/dense", - "interval_solve_d5/near_singular", - "interval_solve_d5/range_exhausted", - "interval_solve_d5/singular", - "interval_solve_d5/uncertain_dense", - "interval_solve_d6/dense", - "interval_solve_d6/near_singular", - "interval_solve_d6/range_exhausted", - "interval_solve_d6/singular", - "interval_solve_d6/uncertain_dense", "linear_form_d2_batch/origin/construction", "linear_form_d2_batch/origin/prepared", "linear_form_d2_batch/origin/with_construction", @@ -2214,11 +1326,6 @@ required = [ "linear_form_d6_prepared/underflow/difference_endpoints", "linear_form_d6_prepared/underflow/dot", "linear_form_d6_prepared/underflow/dot_endpoints", - "polynomial/coincident_lifted/order4/determinant", - "polynomial/coincident_lifted/order5/determinant", - "polynomial/coincident_lifted/order6/determinant", - "polynomial/coincident_lifted/order7/determinant", - "polynomial/coincident_lifted/order8/determinant", "public_d2/det_bound", "public_d2/det_direct", "public_d2/det_with_bound", @@ -2303,108 +1410,1026 @@ required = [ "public_d5/vector_access", "public_d5/vector_new", "public_d5/vector_zero", - "public_exact_d2/matrix_access", - "public_exact_d2/matrix_from_fn", - "public_exact_d2/matrix_from_rows", - "public_exact_d2/matrix_into_rows", - "public_exact_d2/matrix_zero", - "public_exact_d2/vector_access", - "public_exact_d2/vector_from_fn", - "public_exact_d2/vector_into_array", - "public_exact_d2/vector_new", - "public_exact_d2/vector_zero", - "public_exact_d3/matrix_access", - "public_exact_d3/matrix_from_fn", - "public_exact_d3/matrix_from_rows", - "public_exact_d3/matrix_into_rows", - "public_exact_d3/matrix_zero", - "public_exact_d3/vector_access", - "public_exact_d3/vector_from_fn", - "public_exact_d3/vector_into_array", - "public_exact_d3/vector_new", - "public_exact_d3/vector_zero", - "public_exact_d4/matrix_access", - "public_exact_d4/matrix_from_fn", - "public_exact_d4/matrix_from_rows", - "public_exact_d4/matrix_into_rows", - "public_exact_d4/matrix_zero", - "public_exact_d4/vector_access", - "public_exact_d4/vector_from_fn", - "public_exact_d4/vector_into_array", - "public_exact_d4/vector_new", - "public_exact_d4/vector_zero", - "public_exact_d5/matrix_access", - "public_exact_d5/matrix_from_fn", - "public_exact_d5/matrix_from_rows", - "public_exact_d5/matrix_into_rows", - "public_exact_d5/matrix_zero", - "public_exact_d5/vector_access", - "public_exact_d5/vector_from_fn", - "public_exact_d5/vector_into_array", - "public_exact_d5/vector_new", - "public_exact_d5/vector_zero", - "public_exact_scalar/rounded", - "public_exact_scalar/strict", "public_scalar/interval_access", "public_scalar/interval_negate", "public_scalar/interval_new", "public_scalar/interval_point", "public_scalar/interval_subtraction", "public_scalar/tolerance", - "rational_dot/cancellation/d14/prepared", - "rational_dot/cancellation/d14/slice", - "rational_dot/cancellation/d2/prepared", - "rational_dot/cancellation/d2/slice", - "rational_dot/cancellation/d22/prepared", - "rational_dot/cancellation/d22/slice", - "rational_dot/cancellation/d3/prepared", - "rational_dot/cancellation/d3/slice", - "rational_dot/cancellation/d4/prepared", - "rational_dot/cancellation/d4/slice", - "rational_dot/cancellation/d5/prepared", - "rational_dot/cancellation/d5/slice", - "rational_dot/cancellation/d6/prepared", - "rational_dot/cancellation/d6/slice", - "rational_dot/cancellation/d7/prepared", - "rational_dot/cancellation/d7/slice", - "rational_dot/cancellation/d8/prepared", - "rational_dot/cancellation/d8/slice", - "rational_dot/large/d14/prepared", - "rational_dot/large/d14/slice", - "rational_dot/large/d2/prepared", - "rational_dot/large/d2/slice", - "rational_dot/large/d22/prepared", - "rational_dot/large/d22/slice", - "rational_dot/large/d3/prepared", - "rational_dot/large/d3/slice", - "rational_dot/large/d4/prepared", - "rational_dot/large/d4/slice", - "rational_dot/large/d5/prepared", - "rational_dot/large/d5/slice", - "rational_dot/large/d6/prepared", - "rational_dot/large/d6/slice", - "rational_dot/large/d7/prepared", - "rational_dot/large/d7/slice", - "rational_dot/large/d8/prepared", - "rational_dot/large/d8/slice", - "rational_dot/non_dyadic/d14/prepared", - "rational_dot/non_dyadic/d14/slice", - "rational_dot/non_dyadic/d2/prepared", - "rational_dot/non_dyadic/d2/slice", - "rational_dot/non_dyadic/d22/prepared", - "rational_dot/non_dyadic/d22/slice", - "rational_dot/non_dyadic/d3/prepared", - "rational_dot/non_dyadic/d3/slice", - "rational_dot/non_dyadic/d4/prepared", - "rational_dot/non_dyadic/d4/slice", - "rational_dot/non_dyadic/d5/prepared", - "rational_dot/non_dyadic/d5/slice", - "rational_dot/non_dyadic/d6/prepared", - "rational_dot/non_dyadic/d6/slice", - "rational_dot/non_dyadic/d7/prepared", - "rational_dot/non_dyadic/d7/slice", - "rational_dot/non_dyadic/d8/prepared", - "rational_dot/non_dyadic/d8/slice", +] + +[[suites.measurement.phases.current.discovery.suites]] +name = "core-default" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "core-default", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" +phase = "baseline" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites]] +name = "core-exact" +contract = "la-stack-core-exact-v1" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "core-exact" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "common-public-exact" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ + "canonical_conversion_below_overflow_d2/rounded_array", + "canonical_conversion_below_overflow_d2/rounded_result", + "canonical_conversion_below_overflow_d2/strict_array", + "canonical_conversion_below_overflow_d2/strict_result", + "canonical_conversion_below_overflow_d3/rounded_array", + "canonical_conversion_below_overflow_d3/rounded_result", + "canonical_conversion_below_overflow_d3/strict_array", + "canonical_conversion_below_overflow_d3/strict_result", + "canonical_conversion_below_overflow_d4/rounded_array", + "canonical_conversion_below_overflow_d4/rounded_result", + "canonical_conversion_below_overflow_d4/strict_array", + "canonical_conversion_below_overflow_d4/strict_result", + "canonical_conversion_below_overflow_d5/rounded_array", + "canonical_conversion_below_overflow_d5/rounded_result", + "canonical_conversion_below_overflow_d5/strict_array", + "canonical_conversion_below_overflow_d5/strict_result", + "canonical_conversion_dyadic_d2/rounded_array", + "canonical_conversion_dyadic_d2/rounded_result", + "canonical_conversion_dyadic_d2/strict_array", + "canonical_conversion_dyadic_d2/strict_result", + "canonical_conversion_dyadic_d3/rounded_array", + "canonical_conversion_dyadic_d3/rounded_result", + "canonical_conversion_dyadic_d3/strict_array", + "canonical_conversion_dyadic_d3/strict_result", + "canonical_conversion_dyadic_d4/rounded_array", + "canonical_conversion_dyadic_d4/rounded_result", + "canonical_conversion_dyadic_d4/strict_array", + "canonical_conversion_dyadic_d4/strict_result", + "canonical_conversion_dyadic_d5/rounded_array", + "canonical_conversion_dyadic_d5/rounded_result", + "canonical_conversion_dyadic_d5/strict_array", + "canonical_conversion_dyadic_d5/strict_result", + "canonical_conversion_min_subnormal_d2/rounded_array", + "canonical_conversion_min_subnormal_d2/rounded_result", + "canonical_conversion_min_subnormal_d2/strict_array", + "canonical_conversion_min_subnormal_d2/strict_result", + "canonical_conversion_min_subnormal_d3/rounded_array", + "canonical_conversion_min_subnormal_d3/rounded_result", + "canonical_conversion_min_subnormal_d3/strict_array", + "canonical_conversion_min_subnormal_d3/strict_result", + "canonical_conversion_min_subnormal_d4/rounded_array", + "canonical_conversion_min_subnormal_d4/rounded_result", + "canonical_conversion_min_subnormal_d4/strict_array", + "canonical_conversion_min_subnormal_d4/strict_result", + "canonical_conversion_min_subnormal_d5/rounded_array", + "canonical_conversion_min_subnormal_d5/rounded_result", + "canonical_conversion_min_subnormal_d5/strict_array", + "canonical_conversion_min_subnormal_d5/strict_result", + "canonical_conversion_negative_underflow_d2/rounded_array", + "canonical_conversion_negative_underflow_d2/rounded_result", + "canonical_conversion_negative_underflow_d2/strict_array", + "canonical_conversion_negative_underflow_d2/strict_result", + "canonical_conversion_negative_underflow_d3/rounded_array", + "canonical_conversion_negative_underflow_d3/rounded_result", + "canonical_conversion_negative_underflow_d3/strict_array", + "canonical_conversion_negative_underflow_d3/strict_result", + "canonical_conversion_negative_underflow_d4/rounded_array", + "canonical_conversion_negative_underflow_d4/rounded_result", + "canonical_conversion_negative_underflow_d4/strict_array", + "canonical_conversion_negative_underflow_d4/strict_result", + "canonical_conversion_negative_underflow_d5/rounded_array", + "canonical_conversion_negative_underflow_d5/rounded_result", + "canonical_conversion_negative_underflow_d5/strict_array", + "canonical_conversion_negative_underflow_d5/strict_result", + "canonical_conversion_non_dyadic_d2/rounded_array", + "canonical_conversion_non_dyadic_d2/rounded_result", + "canonical_conversion_non_dyadic_d2/strict_array", + "canonical_conversion_non_dyadic_d2/strict_result", + "canonical_conversion_non_dyadic_d3/rounded_array", + "canonical_conversion_non_dyadic_d3/rounded_result", + "canonical_conversion_non_dyadic_d3/strict_array", + "canonical_conversion_non_dyadic_d3/strict_result", + "canonical_conversion_non_dyadic_d4/rounded_array", + "canonical_conversion_non_dyadic_d4/rounded_result", + "canonical_conversion_non_dyadic_d4/strict_array", + "canonical_conversion_non_dyadic_d4/strict_result", + "canonical_conversion_non_dyadic_d5/rounded_array", + "canonical_conversion_non_dyadic_d5/rounded_result", + "canonical_conversion_non_dyadic_d5/strict_array", + "canonical_conversion_non_dyadic_d5/strict_result", + "canonical_conversion_overflow_midpoint_d2/rounded_array", + "canonical_conversion_overflow_midpoint_d2/rounded_result", + "canonical_conversion_overflow_midpoint_d2/strict_array", + "canonical_conversion_overflow_midpoint_d2/strict_result", + "canonical_conversion_overflow_midpoint_d3/rounded_array", + "canonical_conversion_overflow_midpoint_d3/rounded_result", + "canonical_conversion_overflow_midpoint_d3/strict_array", + "canonical_conversion_overflow_midpoint_d3/strict_result", + "canonical_conversion_overflow_midpoint_d4/rounded_array", + "canonical_conversion_overflow_midpoint_d4/rounded_result", + "canonical_conversion_overflow_midpoint_d4/strict_array", + "canonical_conversion_overflow_midpoint_d4/strict_result", + "canonical_conversion_overflow_midpoint_d5/rounded_array", + "canonical_conversion_overflow_midpoint_d5/rounded_result", + "canonical_conversion_overflow_midpoint_d5/strict_array", + "canonical_conversion_overflow_midpoint_d5/strict_result", + "canonical_conversion_wide1024_d2/rounded_array", + "canonical_conversion_wide1024_d2/rounded_result", + "canonical_conversion_wide1024_d2/strict_array", + "canonical_conversion_wide1024_d2/strict_result", + "canonical_conversion_wide1024_d3/rounded_array", + "canonical_conversion_wide1024_d3/rounded_result", + "canonical_conversion_wide1024_d3/strict_array", + "canonical_conversion_wide1024_d3/strict_result", + "canonical_conversion_wide1024_d4/rounded_array", + "canonical_conversion_wide1024_d4/rounded_result", + "canonical_conversion_wide1024_d4/strict_array", + "canonical_conversion_wide1024_d4/strict_result", + "canonical_conversion_wide1024_d5/rounded_array", + "canonical_conversion_wide1024_d5/rounded_result", + "canonical_conversion_wide1024_d5/strict_array", + "canonical_conversion_wide1024_d5/strict_result", + "canonical_conversion_wide256_d2/rounded_array", + "canonical_conversion_wide256_d2/rounded_result", + "canonical_conversion_wide256_d2/strict_array", + "canonical_conversion_wide256_d2/strict_result", + "canonical_conversion_wide256_d3/rounded_array", + "canonical_conversion_wide256_d3/rounded_result", + "canonical_conversion_wide256_d3/strict_array", + "canonical_conversion_wide256_d3/strict_result", + "canonical_conversion_wide256_d4/rounded_array", + "canonical_conversion_wide256_d4/rounded_result", + "canonical_conversion_wide256_d4/strict_array", + "canonical_conversion_wide256_d4/strict_result", + "canonical_conversion_wide256_d5/rounded_array", + "canonical_conversion_wide256_d5/rounded_result", + "canonical_conversion_wide256_d5/strict_array", + "canonical_conversion_wide256_d5/strict_result", + "exact_d2/det", + "exact_d2/det_direct", + "exact_d2/det_direct_with_errbound", + "exact_d2/det_errbound", + "exact_d2/det_exact", + "exact_d2/det_exact_f64_result", + "exact_d2/det_exact_rounded_f64", + "exact_d2/det_sign_exact", + "exact_d2/solve_exact", + "exact_d2/solve_exact_f64_result", + "exact_d2/solve_exact_rounded_f64", + "exact_d3/det", + "exact_d3/det_direct", + "exact_d3/det_direct_with_errbound", + "exact_d3/det_errbound", + "exact_d3/det_exact", + "exact_d3/det_exact_f64_result", + "exact_d3/det_exact_rounded_f64", + "exact_d3/det_sign_exact", + "exact_d3/solve_exact", + "exact_d3/solve_exact_f64_result", + "exact_d3/solve_exact_rounded_f64", + "exact_d4/det", + "exact_d4/det_direct", + "exact_d4/det_direct_with_errbound", + "exact_d4/det_errbound", + "exact_d4/det_exact", + "exact_d4/det_exact_f64_result", + "exact_d4/det_exact_rounded_f64", + "exact_d4/det_sign_exact", + "exact_d4/solve_exact", + "exact_d4/solve_exact_f64_result", + "exact_d4/solve_exact_rounded_f64", + "exact_d5/det", + "exact_d5/det_exact", + "exact_d5/det_exact_f64_result", + "exact_d5/det_exact_rounded_f64", + "exact_d5/det_sign_exact", + "exact_d5/solve_exact", + "exact_d5/solve_exact_f64_result", + "exact_d5/solve_exact_rounded_f64", + "exact_hilbert_4x4/det_exact", + "exact_hilbert_4x4/det_sign_exact", + "exact_hilbert_4x4/solve_exact", + "exact_hilbert_4x4/solve_exact_f64_result", + "exact_hilbert_4x4/solve_exact_rounded_f64", + "exact_hilbert_5x5/det_exact", + "exact_hilbert_5x5/det_sign_exact", + "exact_hilbert_5x5/solve_exact", + "exact_hilbert_5x5/solve_exact_f64_result", + "exact_hilbert_5x5/solve_exact_rounded_f64", + "exact_large_entries_3x3/det_exact", + "exact_large_entries_3x3/det_sign_exact", + "exact_large_entries_3x3/solve_exact", + "exact_large_entries_3x3/solve_exact_f64_result", + "exact_large_entries_3x3/solve_exact_rounded_f64", + "exact_near_singular_3x3/det_exact", + "exact_near_singular_3x3/det_sign_exact", + "exact_near_singular_3x3/solve_exact", + "exact_near_singular_3x3/solve_exact_f64_result", + "exact_near_singular_3x3/solve_exact_rounded_f64", + "exact_random_corpus_d2/det_exact", + "exact_random_corpus_d2/det_sign_exact", + "exact_random_corpus_d2/solve_exact", + "exact_random_corpus_d2/solve_exact_f64_result", + "exact_random_corpus_d2/solve_exact_rounded_f64", + "exact_random_corpus_d3/det_exact", + "exact_random_corpus_d3/det_sign_exact", + "exact_random_corpus_d3/solve_exact", + "exact_random_corpus_d3/solve_exact_f64_result", + "exact_random_corpus_d3/solve_exact_rounded_f64", + "exact_random_corpus_d4/det_exact", + "exact_random_corpus_d4/det_sign_exact", + "exact_random_corpus_d4/solve_exact", + "exact_random_corpus_d4/solve_exact_f64_result", + "exact_random_corpus_d4/solve_exact_rounded_f64", + "exact_random_corpus_d5/det_exact", + "exact_random_corpus_d5/det_sign_exact", + "exact_random_corpus_d5/solve_exact", + "exact_random_corpus_d5/solve_exact_f64_result", + "exact_random_corpus_d5/solve_exact_rounded_f64", + "public_exact_d2/matrix_access", + "public_exact_d2/matrix_from_fn", + "public_exact_d2/matrix_from_rows", + "public_exact_d2/matrix_into_rows", + "public_exact_d2/matrix_zero", + "public_exact_d2/vector_access", + "public_exact_d2/vector_from_fn", + "public_exact_d2/vector_into_array", + "public_exact_d2/vector_new", + "public_exact_d2/vector_zero", + "public_exact_d3/matrix_access", + "public_exact_d3/matrix_from_fn", + "public_exact_d3/matrix_from_rows", + "public_exact_d3/matrix_into_rows", + "public_exact_d3/matrix_zero", + "public_exact_d3/vector_access", + "public_exact_d3/vector_from_fn", + "public_exact_d3/vector_into_array", + "public_exact_d3/vector_new", + "public_exact_d3/vector_zero", + "public_exact_d4/matrix_access", + "public_exact_d4/matrix_from_fn", + "public_exact_d4/matrix_from_rows", + "public_exact_d4/matrix_into_rows", + "public_exact_d4/matrix_zero", + "public_exact_d4/vector_access", + "public_exact_d4/vector_from_fn", + "public_exact_d4/vector_into_array", + "public_exact_d4/vector_new", + "public_exact_d4/vector_zero", + "public_exact_d5/matrix_access", + "public_exact_d5/matrix_from_fn", + "public_exact_d5/matrix_from_rows", + "public_exact_d5/matrix_into_rows", + "public_exact_d5/matrix_zero", + "public_exact_d5/vector_access", + "public_exact_d5/vector_from_fn", + "public_exact_d5/vector_into_array", + "public_exact_d5/vector_new", + "public_exact_d5/vector_zero", + "public_exact_scalar/rounded", + "public_exact_scalar/strict", + "rational_input_d2/det_row_cleared_bareiss", + "rational_input_d2/det_sign_row_cleared_bareiss", + "rational_input_d2/solve_row_cleared_bareiss", + "rational_input_d3/det_row_cleared_bareiss", + "rational_input_d3/det_sign_row_cleared_bareiss", + "rational_input_d3/solve_row_cleared_bareiss", + "rational_input_d4/det_row_cleared_bareiss", + "rational_input_d4/det_sign_row_cleared_bareiss", + "rational_input_d4/solve_row_cleared_bareiss", + "rational_input_d5/det_row_cleared_bareiss", + "rational_input_d5/det_sign_row_cleared_bareiss", + "rational_input_d5/solve_row_cleared_bareiss", + "rational_input_d6/det_row_cleared_bareiss", + "rational_input_d6/det_sign_row_cleared_bareiss", + "rational_input_d6/solve_row_cleared_bareiss", + "rational_input_d7/det_row_cleared_bareiss", + "rational_input_d7/det_sign_row_cleared_bareiss", + "rational_input_d7/solve_row_cleared_bareiss", + "rational_input_d8/det_row_cleared_bareiss", + "rational_input_d8/det_sign_row_cleared_bareiss", + "rational_input_d8/solve_row_cleared_bareiss", + "rational_input_wide1024_d2/det_row_cleared_bareiss", + "rational_input_wide1024_d2/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d2/solve_row_cleared_bareiss", + "rational_input_wide1024_d3/det_row_cleared_bareiss", + "rational_input_wide1024_d3/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d3/solve_row_cleared_bareiss", + "rational_input_wide1024_d4/det_row_cleared_bareiss", + "rational_input_wide1024_d4/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d4/solve_row_cleared_bareiss", + "rational_input_wide1024_d5/det_row_cleared_bareiss", + "rational_input_wide1024_d5/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d5/solve_row_cleared_bareiss", + "rational_input_wide1024_d6/det_row_cleared_bareiss", + "rational_input_wide1024_d6/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d6/solve_row_cleared_bareiss", + "rational_input_wide1024_d7/det_row_cleared_bareiss", + "rational_input_wide1024_d7/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d7/solve_row_cleared_bareiss", + "rational_input_wide1024_d8/det_row_cleared_bareiss", + "rational_input_wide1024_d8/det_sign_row_cleared_bareiss", + "rational_input_wide1024_d8/solve_row_cleared_bareiss", + "rational_input_wide256_d2/det_row_cleared_bareiss", + "rational_input_wide256_d2/det_sign_row_cleared_bareiss", + "rational_input_wide256_d2/solve_row_cleared_bareiss", + "rational_input_wide256_d3/det_row_cleared_bareiss", + "rational_input_wide256_d3/det_sign_row_cleared_bareiss", + "rational_input_wide256_d3/solve_row_cleared_bareiss", + "rational_input_wide256_d4/det_row_cleared_bareiss", + "rational_input_wide256_d4/det_sign_row_cleared_bareiss", + "rational_input_wide256_d4/solve_row_cleared_bareiss", + "rational_input_wide256_d5/det_row_cleared_bareiss", + "rational_input_wide256_d5/det_sign_row_cleared_bareiss", + "rational_input_wide256_d5/solve_row_cleared_bareiss", + "rational_input_wide256_d6/det_row_cleared_bareiss", + "rational_input_wide256_d6/det_sign_row_cleared_bareiss", + "rational_input_wide256_d6/solve_row_cleared_bareiss", + "rational_input_wide256_d7/det_row_cleared_bareiss", + "rational_input_wide256_d7/det_sign_row_cleared_bareiss", + "rational_input_wide256_d7/solve_row_cleared_bareiss", + "rational_input_wide256_d8/det_row_cleared_bareiss", + "rational_input_wide256_d8/det_sign_row_cleared_bareiss", + "rational_input_wide256_d8/solve_row_cleared_bareiss", + "rational_solve/dense/dyadic_d2/adapter", + "rational_solve/dense/dyadic_d2/construct", + "rational_solve/dense/dyadic_d2/prepared", + "rational_solve/dense/dyadic_d3/adapter", + "rational_solve/dense/dyadic_d3/construct", + "rational_solve/dense/dyadic_d3/prepared", + "rational_solve/dense/dyadic_d4/adapter", + "rational_solve/dense/dyadic_d4/construct", + "rational_solve/dense/dyadic_d4/prepared", + "rational_solve/dense/dyadic_d5/adapter", + "rational_solve/dense/dyadic_d5/construct", + "rational_solve/dense/dyadic_d5/prepared", + "rational_solve/dense/dyadic_d6/adapter", + "rational_solve/dense/dyadic_d6/construct", + "rational_solve/dense/dyadic_d6/prepared", + "rational_solve/dense/dyadic_d7/adapter", + "rational_solve/dense/dyadic_d7/construct", + "rational_solve/dense/dyadic_d7/prepared", + "rational_solve/dense/dyadic_d8/adapter", + "rational_solve/dense/dyadic_d8/construct", + "rational_solve/dense/dyadic_d8/prepared", + "rational_solve/dense/rational_d2/adapter", + "rational_solve/dense/rational_d2/construct", + "rational_solve/dense/rational_d2/prepared", + "rational_solve/dense/rational_d3/adapter", + "rational_solve/dense/rational_d3/construct", + "rational_solve/dense/rational_d3/prepared", + "rational_solve/dense/rational_d4/adapter", + "rational_solve/dense/rational_d4/construct", + "rational_solve/dense/rational_d4/prepared", + "rational_solve/dense/rational_d5/adapter", + "rational_solve/dense/rational_d5/construct", + "rational_solve/dense/rational_d5/prepared", + "rational_solve/dense/rational_d6/adapter", + "rational_solve/dense/rational_d6/construct", + "rational_solve/dense/rational_d6/prepared", + "rational_solve/dense/rational_d7/adapter", + "rational_solve/dense/rational_d7/construct", + "rational_solve/dense/rational_d7/prepared", + "rational_solve/dense/rational_d8/adapter", + "rational_solve/dense/rational_d8/construct", + "rational_solve/dense/rational_d8/prepared", + "rational_solve/diagonal/dyadic_d2/adapter", + "rational_solve/diagonal/dyadic_d2/construct", + "rational_solve/diagonal/dyadic_d2/prepared", + "rational_solve/diagonal/dyadic_d3/adapter", + "rational_solve/diagonal/dyadic_d3/construct", + "rational_solve/diagonal/dyadic_d3/prepared", + "rational_solve/diagonal/dyadic_d4/adapter", + "rational_solve/diagonal/dyadic_d4/construct", + "rational_solve/diagonal/dyadic_d4/prepared", + "rational_solve/diagonal/dyadic_d5/adapter", + "rational_solve/diagonal/dyadic_d5/construct", + "rational_solve/diagonal/dyadic_d5/prepared", + "rational_solve/diagonal/dyadic_d6/adapter", + "rational_solve/diagonal/dyadic_d6/construct", + "rational_solve/diagonal/dyadic_d6/prepared", + "rational_solve/diagonal/dyadic_d7/adapter", + "rational_solve/diagonal/dyadic_d7/construct", + "rational_solve/diagonal/dyadic_d7/prepared", + "rational_solve/diagonal/dyadic_d8/adapter", + "rational_solve/diagonal/dyadic_d8/construct", + "rational_solve/diagonal/dyadic_d8/prepared", + "rational_solve/diagonal/rational_d2/adapter", + "rational_solve/diagonal/rational_d2/construct", + "rational_solve/diagonal/rational_d2/prepared", + "rational_solve/diagonal/rational_d3/adapter", + "rational_solve/diagonal/rational_d3/construct", + "rational_solve/diagonal/rational_d3/prepared", + "rational_solve/diagonal/rational_d4/adapter", + "rational_solve/diagonal/rational_d4/construct", + "rational_solve/diagonal/rational_d4/prepared", + "rational_solve/diagonal/rational_d5/adapter", + "rational_solve/diagonal/rational_d5/construct", + "rational_solve/diagonal/rational_d5/prepared", + "rational_solve/diagonal/rational_d6/adapter", + "rational_solve/diagonal/rational_d6/construct", + "rational_solve/diagonal/rational_d6/prepared", + "rational_solve/diagonal/rational_d7/adapter", + "rational_solve/diagonal/rational_d7/construct", + "rational_solve/diagonal/rational_d7/prepared", + "rational_solve/diagonal/rational_d8/adapter", + "rational_solve/diagonal/rational_d8/construct", + "rational_solve/diagonal/rational_d8/prepared", + "rational_solve/lower/dyadic_d2/adapter", + "rational_solve/lower/dyadic_d2/construct", + "rational_solve/lower/dyadic_d2/prepared", + "rational_solve/lower/dyadic_d3/adapter", + "rational_solve/lower/dyadic_d3/construct", + "rational_solve/lower/dyadic_d3/prepared", + "rational_solve/lower/dyadic_d4/adapter", + "rational_solve/lower/dyadic_d4/construct", + "rational_solve/lower/dyadic_d4/prepared", + "rational_solve/lower/dyadic_d5/adapter", + "rational_solve/lower/dyadic_d5/construct", + "rational_solve/lower/dyadic_d5/prepared", + "rational_solve/lower/dyadic_d6/adapter", + "rational_solve/lower/dyadic_d6/construct", + "rational_solve/lower/dyadic_d6/prepared", + "rational_solve/lower/dyadic_d7/adapter", + "rational_solve/lower/dyadic_d7/construct", + "rational_solve/lower/dyadic_d7/prepared", + "rational_solve/lower/dyadic_d8/adapter", + "rational_solve/lower/dyadic_d8/construct", + "rational_solve/lower/dyadic_d8/prepared", + "rational_solve/lower/rational_d2/adapter", + "rational_solve/lower/rational_d2/construct", + "rational_solve/lower/rational_d2/prepared", + "rational_solve/lower/rational_d3/adapter", + "rational_solve/lower/rational_d3/construct", + "rational_solve/lower/rational_d3/prepared", + "rational_solve/lower/rational_d4/adapter", + "rational_solve/lower/rational_d4/construct", + "rational_solve/lower/rational_d4/prepared", + "rational_solve/lower/rational_d5/adapter", + "rational_solve/lower/rational_d5/construct", + "rational_solve/lower/rational_d5/prepared", + "rational_solve/lower/rational_d6/adapter", + "rational_solve/lower/rational_d6/construct", + "rational_solve/lower/rational_d6/prepared", + "rational_solve/lower/rational_d7/adapter", + "rational_solve/lower/rational_d7/construct", + "rational_solve/lower/rational_d7/prepared", + "rational_solve/lower/rational_d8/adapter", + "rational_solve/lower/rational_d8/construct", + "rational_solve/lower/rational_d8/prepared", + "rational_solve/permuted_lower/dyadic_d2/adapter", + "rational_solve/permuted_lower/dyadic_d2/construct", + "rational_solve/permuted_lower/dyadic_d2/prepared", + "rational_solve/permuted_lower/dyadic_d3/adapter", + "rational_solve/permuted_lower/dyadic_d3/construct", + "rational_solve/permuted_lower/dyadic_d3/prepared", + "rational_solve/permuted_lower/dyadic_d4/adapter", + "rational_solve/permuted_lower/dyadic_d4/construct", + "rational_solve/permuted_lower/dyadic_d4/prepared", + "rational_solve/permuted_lower/dyadic_d5/adapter", + "rational_solve/permuted_lower/dyadic_d5/construct", + "rational_solve/permuted_lower/dyadic_d5/prepared", + "rational_solve/permuted_lower/dyadic_d6/adapter", + "rational_solve/permuted_lower/dyadic_d6/construct", + "rational_solve/permuted_lower/dyadic_d6/prepared", + "rational_solve/permuted_lower/dyadic_d7/adapter", + "rational_solve/permuted_lower/dyadic_d7/construct", + "rational_solve/permuted_lower/dyadic_d7/prepared", + "rational_solve/permuted_lower/dyadic_d8/adapter", + "rational_solve/permuted_lower/dyadic_d8/construct", + "rational_solve/permuted_lower/dyadic_d8/prepared", + "rational_solve/permuted_lower/rational_d2/adapter", + "rational_solve/permuted_lower/rational_d2/construct", + "rational_solve/permuted_lower/rational_d2/prepared", + "rational_solve/permuted_lower/rational_d3/adapter", + "rational_solve/permuted_lower/rational_d3/construct", + "rational_solve/permuted_lower/rational_d3/prepared", + "rational_solve/permuted_lower/rational_d4/adapter", + "rational_solve/permuted_lower/rational_d4/construct", + "rational_solve/permuted_lower/rational_d4/prepared", + "rational_solve/permuted_lower/rational_d5/adapter", + "rational_solve/permuted_lower/rational_d5/construct", + "rational_solve/permuted_lower/rational_d5/prepared", + "rational_solve/permuted_lower/rational_d6/adapter", + "rational_solve/permuted_lower/rational_d6/construct", + "rational_solve/permuted_lower/rational_d6/prepared", + "rational_solve/permuted_lower/rational_d7/adapter", + "rational_solve/permuted_lower/rational_d7/construct", + "rational_solve/permuted_lower/rational_d7/prepared", + "rational_solve/permuted_lower/rational_d8/adapter", + "rational_solve/permuted_lower/rational_d8/construct", + "rational_solve/permuted_lower/rational_d8/prepared", + "rational_solve/permuted_upper/dyadic_d2/adapter", + "rational_solve/permuted_upper/dyadic_d2/construct", + "rational_solve/permuted_upper/dyadic_d2/prepared", + "rational_solve/permuted_upper/dyadic_d3/adapter", + "rational_solve/permuted_upper/dyadic_d3/construct", + "rational_solve/permuted_upper/dyadic_d3/prepared", + "rational_solve/permuted_upper/dyadic_d4/adapter", + "rational_solve/permuted_upper/dyadic_d4/construct", + "rational_solve/permuted_upper/dyadic_d4/prepared", + "rational_solve/permuted_upper/dyadic_d5/adapter", + "rational_solve/permuted_upper/dyadic_d5/construct", + "rational_solve/permuted_upper/dyadic_d5/prepared", + "rational_solve/permuted_upper/dyadic_d6/adapter", + "rational_solve/permuted_upper/dyadic_d6/construct", + "rational_solve/permuted_upper/dyadic_d6/prepared", + "rational_solve/permuted_upper/dyadic_d7/adapter", + "rational_solve/permuted_upper/dyadic_d7/construct", + "rational_solve/permuted_upper/dyadic_d7/prepared", + "rational_solve/permuted_upper/dyadic_d8/adapter", + "rational_solve/permuted_upper/dyadic_d8/construct", + "rational_solve/permuted_upper/dyadic_d8/prepared", + "rational_solve/permuted_upper/rational_d2/adapter", + "rational_solve/permuted_upper/rational_d2/construct", + "rational_solve/permuted_upper/rational_d2/prepared", + "rational_solve/permuted_upper/rational_d3/adapter", + "rational_solve/permuted_upper/rational_d3/construct", + "rational_solve/permuted_upper/rational_d3/prepared", + "rational_solve/permuted_upper/rational_d4/adapter", + "rational_solve/permuted_upper/rational_d4/construct", + "rational_solve/permuted_upper/rational_d4/prepared", + "rational_solve/permuted_upper/rational_d5/adapter", + "rational_solve/permuted_upper/rational_d5/construct", + "rational_solve/permuted_upper/rational_d5/prepared", + "rational_solve/permuted_upper/rational_d6/adapter", + "rational_solve/permuted_upper/rational_d6/construct", + "rational_solve/permuted_upper/rational_d6/prepared", + "rational_solve/permuted_upper/rational_d7/adapter", + "rational_solve/permuted_upper/rational_d7/construct", + "rational_solve/permuted_upper/rational_d7/prepared", + "rational_solve/permuted_upper/rational_d8/adapter", + "rational_solve/permuted_upper/rational_d8/construct", + "rational_solve/permuted_upper/rational_d8/prepared", + "rational_solve/sparse/dyadic_d2/adapter", + "rational_solve/sparse/dyadic_d2/construct", + "rational_solve/sparse/dyadic_d2/prepared", + "rational_solve/sparse/dyadic_d3/adapter", + "rational_solve/sparse/dyadic_d3/construct", + "rational_solve/sparse/dyadic_d3/prepared", + "rational_solve/sparse/dyadic_d4/adapter", + "rational_solve/sparse/dyadic_d4/construct", + "rational_solve/sparse/dyadic_d4/prepared", + "rational_solve/sparse/dyadic_d5/adapter", + "rational_solve/sparse/dyadic_d5/construct", + "rational_solve/sparse/dyadic_d5/prepared", + "rational_solve/sparse/dyadic_d6/adapter", + "rational_solve/sparse/dyadic_d6/construct", + "rational_solve/sparse/dyadic_d6/prepared", + "rational_solve/sparse/dyadic_d7/adapter", + "rational_solve/sparse/dyadic_d7/construct", + "rational_solve/sparse/dyadic_d7/prepared", + "rational_solve/sparse/dyadic_d8/adapter", + "rational_solve/sparse/dyadic_d8/construct", + "rational_solve/sparse/dyadic_d8/prepared", + "rational_solve/sparse/rational_d2/adapter", + "rational_solve/sparse/rational_d2/construct", + "rational_solve/sparse/rational_d2/prepared", + "rational_solve/sparse/rational_d3/adapter", + "rational_solve/sparse/rational_d3/construct", + "rational_solve/sparse/rational_d3/prepared", + "rational_solve/sparse/rational_d4/adapter", + "rational_solve/sparse/rational_d4/construct", + "rational_solve/sparse/rational_d4/prepared", + "rational_solve/sparse/rational_d5/adapter", + "rational_solve/sparse/rational_d5/construct", + "rational_solve/sparse/rational_d5/prepared", + "rational_solve/sparse/rational_d6/adapter", + "rational_solve/sparse/rational_d6/construct", + "rational_solve/sparse/rational_d6/prepared", + "rational_solve/sparse/rational_d7/adapter", + "rational_solve/sparse/rational_d7/construct", + "rational_solve/sparse/rational_d7/prepared", + "rational_solve/sparse/rational_d8/adapter", + "rational_solve/sparse/rational_d8/construct", + "rational_solve/sparse/rational_d8/prepared", + "rational_solve/upper/dyadic_d2/adapter", + "rational_solve/upper/dyadic_d2/construct", + "rational_solve/upper/dyadic_d2/prepared", + "rational_solve/upper/dyadic_d3/adapter", + "rational_solve/upper/dyadic_d3/construct", + "rational_solve/upper/dyadic_d3/prepared", + "rational_solve/upper/dyadic_d4/adapter", + "rational_solve/upper/dyadic_d4/construct", + "rational_solve/upper/dyadic_d4/prepared", + "rational_solve/upper/dyadic_d5/adapter", + "rational_solve/upper/dyadic_d5/construct", + "rational_solve/upper/dyadic_d5/prepared", + "rational_solve/upper/dyadic_d6/adapter", + "rational_solve/upper/dyadic_d6/construct", + "rational_solve/upper/dyadic_d6/prepared", + "rational_solve/upper/dyadic_d7/adapter", + "rational_solve/upper/dyadic_d7/construct", + "rational_solve/upper/dyadic_d7/prepared", + "rational_solve/upper/dyadic_d8/adapter", + "rational_solve/upper/dyadic_d8/construct", + "rational_solve/upper/dyadic_d8/prepared", + "rational_solve/upper/rational_d2/adapter", + "rational_solve/upper/rational_d2/construct", + "rational_solve/upper/rational_d2/prepared", + "rational_solve/upper/rational_d3/adapter", + "rational_solve/upper/rational_d3/construct", + "rational_solve/upper/rational_d3/prepared", + "rational_solve/upper/rational_d4/adapter", + "rational_solve/upper/rational_d4/construct", + "rational_solve/upper/rational_d4/prepared", + "rational_solve/upper/rational_d5/adapter", + "rational_solve/upper/rational_d5/construct", + "rational_solve/upper/rational_d5/prepared", + "rational_solve/upper/rational_d6/adapter", + "rational_solve/upper/rational_d6/construct", + "rational_solve/upper/rational_d6/prepared", + "rational_solve/upper/rational_d7/adapter", + "rational_solve/upper/rational_d7/construct", + "rational_solve/upper/rational_d7/prepared", + "rational_solve/upper/rational_d8/adapter", + "rational_solve/upper/rational_d8/construct", + "rational_solve/upper/rational_d8/prepared", +] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "core-exact" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "common-public-exact", + "list", +] +timeout = 14400 + +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "common-public-exact" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ + "canonical_conversion_below_overflow_d2/rounded_array", + "canonical_conversion_below_overflow_d2/rounded_result", + "canonical_conversion_below_overflow_d2/strict_array", + "canonical_conversion_below_overflow_d2/strict_result", + "canonical_conversion_below_overflow_d3/rounded_array", + "canonical_conversion_below_overflow_d3/rounded_result", + "canonical_conversion_below_overflow_d3/strict_array", + "canonical_conversion_below_overflow_d3/strict_result", + "canonical_conversion_below_overflow_d4/rounded_array", + "canonical_conversion_below_overflow_d4/rounded_result", + "canonical_conversion_below_overflow_d4/strict_array", + "canonical_conversion_below_overflow_d4/strict_result", + "canonical_conversion_below_overflow_d5/rounded_array", + "canonical_conversion_below_overflow_d5/rounded_result", + "canonical_conversion_below_overflow_d5/strict_array", + "canonical_conversion_below_overflow_d5/strict_result", + "canonical_conversion_dyadic_d2/rounded_array", + "canonical_conversion_dyadic_d2/rounded_result", + "canonical_conversion_dyadic_d2/strict_array", + "canonical_conversion_dyadic_d2/strict_result", + "canonical_conversion_dyadic_d3/rounded_array", + "canonical_conversion_dyadic_d3/rounded_result", + "canonical_conversion_dyadic_d3/strict_array", + "canonical_conversion_dyadic_d3/strict_result", + "canonical_conversion_dyadic_d4/rounded_array", + "canonical_conversion_dyadic_d4/rounded_result", + "canonical_conversion_dyadic_d4/strict_array", + "canonical_conversion_dyadic_d4/strict_result", + "canonical_conversion_dyadic_d5/rounded_array", + "canonical_conversion_dyadic_d5/rounded_result", + "canonical_conversion_dyadic_d5/strict_array", + "canonical_conversion_dyadic_d5/strict_result", + "canonical_conversion_min_subnormal_d2/rounded_array", + "canonical_conversion_min_subnormal_d2/rounded_result", + "canonical_conversion_min_subnormal_d2/strict_array", + "canonical_conversion_min_subnormal_d2/strict_result", + "canonical_conversion_min_subnormal_d3/rounded_array", + "canonical_conversion_min_subnormal_d3/rounded_result", + "canonical_conversion_min_subnormal_d3/strict_array", + "canonical_conversion_min_subnormal_d3/strict_result", + "canonical_conversion_min_subnormal_d4/rounded_array", + "canonical_conversion_min_subnormal_d4/rounded_result", + "canonical_conversion_min_subnormal_d4/strict_array", + "canonical_conversion_min_subnormal_d4/strict_result", + "canonical_conversion_min_subnormal_d5/rounded_array", + "canonical_conversion_min_subnormal_d5/rounded_result", + "canonical_conversion_min_subnormal_d5/strict_array", + "canonical_conversion_min_subnormal_d5/strict_result", + "canonical_conversion_negative_underflow_d2/rounded_array", + "canonical_conversion_negative_underflow_d2/rounded_result", + "canonical_conversion_negative_underflow_d2/strict_array", + "canonical_conversion_negative_underflow_d2/strict_result", + "canonical_conversion_negative_underflow_d3/rounded_array", + "canonical_conversion_negative_underflow_d3/rounded_result", + "canonical_conversion_negative_underflow_d3/strict_array", + "canonical_conversion_negative_underflow_d3/strict_result", + "canonical_conversion_negative_underflow_d4/rounded_array", + "canonical_conversion_negative_underflow_d4/rounded_result", + "canonical_conversion_negative_underflow_d4/strict_array", + "canonical_conversion_negative_underflow_d4/strict_result", + "canonical_conversion_negative_underflow_d5/rounded_array", + "canonical_conversion_negative_underflow_d5/rounded_result", + "canonical_conversion_negative_underflow_d5/strict_array", + "canonical_conversion_negative_underflow_d5/strict_result", + "canonical_conversion_non_dyadic_d2/rounded_array", + "canonical_conversion_non_dyadic_d2/rounded_result", + "canonical_conversion_non_dyadic_d2/strict_array", + "canonical_conversion_non_dyadic_d2/strict_result", + "canonical_conversion_non_dyadic_d3/rounded_array", + "canonical_conversion_non_dyadic_d3/rounded_result", + "canonical_conversion_non_dyadic_d3/strict_array", + "canonical_conversion_non_dyadic_d3/strict_result", + "canonical_conversion_non_dyadic_d4/rounded_array", + "canonical_conversion_non_dyadic_d4/rounded_result", + "canonical_conversion_non_dyadic_d4/strict_array", + "canonical_conversion_non_dyadic_d4/strict_result", + "canonical_conversion_non_dyadic_d5/rounded_array", + "canonical_conversion_non_dyadic_d5/rounded_result", + "canonical_conversion_non_dyadic_d5/strict_array", + "canonical_conversion_non_dyadic_d5/strict_result", + "canonical_conversion_overflow_midpoint_d2/rounded_array", + "canonical_conversion_overflow_midpoint_d2/rounded_result", + "canonical_conversion_overflow_midpoint_d2/strict_array", + "canonical_conversion_overflow_midpoint_d2/strict_result", + "canonical_conversion_overflow_midpoint_d3/rounded_array", + "canonical_conversion_overflow_midpoint_d3/rounded_result", + "canonical_conversion_overflow_midpoint_d3/strict_array", + "canonical_conversion_overflow_midpoint_d3/strict_result", + "canonical_conversion_overflow_midpoint_d4/rounded_array", + "canonical_conversion_overflow_midpoint_d4/rounded_result", + "canonical_conversion_overflow_midpoint_d4/strict_array", + "canonical_conversion_overflow_midpoint_d4/strict_result", + "canonical_conversion_overflow_midpoint_d5/rounded_array", + "canonical_conversion_overflow_midpoint_d5/rounded_result", + "canonical_conversion_overflow_midpoint_d5/strict_array", + "canonical_conversion_overflow_midpoint_d5/strict_result", + "canonical_conversion_wide1024_d2/rounded_array", + "canonical_conversion_wide1024_d2/rounded_result", + "canonical_conversion_wide1024_d2/strict_array", + "canonical_conversion_wide1024_d2/strict_result", + "canonical_conversion_wide1024_d3/rounded_array", + "canonical_conversion_wide1024_d3/rounded_result", + "canonical_conversion_wide1024_d3/strict_array", + "canonical_conversion_wide1024_d3/strict_result", + "canonical_conversion_wide1024_d4/rounded_array", + "canonical_conversion_wide1024_d4/rounded_result", + "canonical_conversion_wide1024_d4/strict_array", + "canonical_conversion_wide1024_d4/strict_result", + "canonical_conversion_wide1024_d5/rounded_array", + "canonical_conversion_wide1024_d5/rounded_result", + "canonical_conversion_wide1024_d5/strict_array", + "canonical_conversion_wide1024_d5/strict_result", + "canonical_conversion_wide256_d2/rounded_array", + "canonical_conversion_wide256_d2/rounded_result", + "canonical_conversion_wide256_d2/strict_array", + "canonical_conversion_wide256_d2/strict_result", + "canonical_conversion_wide256_d3/rounded_array", + "canonical_conversion_wide256_d3/rounded_result", + "canonical_conversion_wide256_d3/strict_array", + "canonical_conversion_wide256_d3/strict_result", + "canonical_conversion_wide256_d4/rounded_array", + "canonical_conversion_wide256_d4/rounded_result", + "canonical_conversion_wide256_d4/strict_array", + "canonical_conversion_wide256_d4/strict_result", + "canonical_conversion_wide256_d5/rounded_array", + "canonical_conversion_wide256_d5/rounded_result", + "canonical_conversion_wide256_d5/strict_array", + "canonical_conversion_wide256_d5/strict_result", + "exact_d2/det", + "exact_d2/det_direct", + "exact_d2/det_direct_with_errbound", + "exact_d2/det_errbound", + "exact_d2/det_exact", + "exact_d2/det_exact_f64_result", + "exact_d2/det_exact_rounded_f64", + "exact_d2/det_sign_exact", + "exact_d2/solve_exact", + "exact_d2/solve_exact_f64_result", + "exact_d2/solve_exact_rounded_f64", + "exact_d3/det", + "exact_d3/det_direct", + "exact_d3/det_direct_with_errbound", + "exact_d3/det_errbound", + "exact_d3/det_exact", + "exact_d3/det_exact_f64_result", + "exact_d3/det_exact_rounded_f64", + "exact_d3/det_sign_exact", + "exact_d3/solve_exact", + "exact_d3/solve_exact_f64_result", + "exact_d3/solve_exact_rounded_f64", + "exact_d4/det", + "exact_d4/det_direct", + "exact_d4/det_direct_with_errbound", + "exact_d4/det_errbound", + "exact_d4/det_exact", + "exact_d4/det_exact_f64_result", + "exact_d4/det_exact_rounded_f64", + "exact_d4/det_sign_exact", + "exact_d4/solve_exact", + "exact_d4/solve_exact_f64_result", + "exact_d4/solve_exact_rounded_f64", + "exact_d5/det", + "exact_d5/det_exact", + "exact_d5/det_exact_f64_result", + "exact_d5/det_exact_rounded_f64", + "exact_d5/det_sign_exact", + "exact_d5/solve_exact", + "exact_d5/solve_exact_f64_result", + "exact_d5/solve_exact_rounded_f64", + "exact_hilbert_4x4/det_exact", + "exact_hilbert_4x4/det_sign_exact", + "exact_hilbert_4x4/solve_exact", + "exact_hilbert_4x4/solve_exact_f64_result", + "exact_hilbert_4x4/solve_exact_rounded_f64", + "exact_hilbert_5x5/det_exact", + "exact_hilbert_5x5/det_sign_exact", + "exact_hilbert_5x5/solve_exact", + "exact_hilbert_5x5/solve_exact_f64_result", + "exact_hilbert_5x5/solve_exact_rounded_f64", + "exact_large_entries_3x3/det_exact", + "exact_large_entries_3x3/det_sign_exact", + "exact_large_entries_3x3/solve_exact", + "exact_large_entries_3x3/solve_exact_f64_result", + "exact_large_entries_3x3/solve_exact_rounded_f64", + "exact_near_singular_3x3/det_exact", + "exact_near_singular_3x3/det_sign_exact", + "exact_near_singular_3x3/solve_exact", + "exact_near_singular_3x3/solve_exact_f64_result", + "exact_near_singular_3x3/solve_exact_rounded_f64", + "exact_random_corpus_d2/det_exact", + "exact_random_corpus_d2/det_sign_exact", + "exact_random_corpus_d2/solve_exact", + "exact_random_corpus_d2/solve_exact_f64_result", + "exact_random_corpus_d2/solve_exact_rounded_f64", + "exact_random_corpus_d3/det_exact", + "exact_random_corpus_d3/det_sign_exact", + "exact_random_corpus_d3/solve_exact", + "exact_random_corpus_d3/solve_exact_f64_result", + "exact_random_corpus_d3/solve_exact_rounded_f64", + "exact_random_corpus_d4/det_exact", + "exact_random_corpus_d4/det_sign_exact", + "exact_random_corpus_d4/solve_exact", + "exact_random_corpus_d4/solve_exact_f64_result", + "exact_random_corpus_d4/solve_exact_rounded_f64", + "exact_random_corpus_d5/det_exact", + "exact_random_corpus_d5/det_sign_exact", + "exact_random_corpus_d5/solve_exact", + "exact_random_corpus_d5/solve_exact_f64_result", + "exact_random_corpus_d5/solve_exact_rounded_f64", + "public_exact_d2/matrix_access", + "public_exact_d2/matrix_from_fn", + "public_exact_d2/matrix_from_rows", + "public_exact_d2/matrix_into_rows", + "public_exact_d2/matrix_zero", + "public_exact_d2/vector_access", + "public_exact_d2/vector_from_fn", + "public_exact_d2/vector_into_array", + "public_exact_d2/vector_new", + "public_exact_d2/vector_zero", + "public_exact_d3/matrix_access", + "public_exact_d3/matrix_from_fn", + "public_exact_d3/matrix_from_rows", + "public_exact_d3/matrix_into_rows", + "public_exact_d3/matrix_zero", + "public_exact_d3/vector_access", + "public_exact_d3/vector_from_fn", + "public_exact_d3/vector_into_array", + "public_exact_d3/vector_new", + "public_exact_d3/vector_zero", + "public_exact_d4/matrix_access", + "public_exact_d4/matrix_from_fn", + "public_exact_d4/matrix_from_rows", + "public_exact_d4/matrix_into_rows", + "public_exact_d4/matrix_zero", + "public_exact_d4/vector_access", + "public_exact_d4/vector_from_fn", + "public_exact_d4/vector_into_array", + "public_exact_d4/vector_new", + "public_exact_d4/vector_zero", + "public_exact_d5/matrix_access", + "public_exact_d5/matrix_from_fn", + "public_exact_d5/matrix_from_rows", + "public_exact_d5/matrix_into_rows", + "public_exact_d5/matrix_zero", + "public_exact_d5/vector_access", + "public_exact_d5/vector_from_fn", + "public_exact_d5/vector_into_array", + "public_exact_d5/vector_new", + "public_exact_d5/vector_zero", + "public_exact_scalar/rounded", + "public_exact_scalar/strict", "rational_input_d2/det_row_cleared_bareiss", "rational_input_d2/det_sign_row_cleared_bareiss", "rational_input_d2/solve_row_cleared_bareiss", @@ -2764,48 +2789,1172 @@ required = [ "rational_solve/upper/rational_d8/prepared", ] -[[phases.current.discovery.suites]] -name = "public-default" -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "current-public-default", "list" ] +[[suites.measurement.phases.current.discovery.suites]] +name = "core-exact" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "common-public-exact", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" +phase = "baseline" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites]] +name = "lu-solve" +contract = "la-stack-lu-solve-v1" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "lu-solve" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "lu-solve" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ + "d16/faer_lu_solve", + "d16/la_stack_lu_solve", + "d16/nalgebra_lu_solve", + "d2/faer_lu_solve", + "d2/la_stack_lu_solve", + "d2/nalgebra_lu_solve", + "d3/faer_lu_solve", + "d3/la_stack_lu_solve", + "d3/nalgebra_lu_solve", + "d32/faer_lu_solve", + "d32/la_stack_lu_solve", + "d32/nalgebra_lu_solve", + "d4/faer_lu_solve", + "d4/la_stack_lu_solve", + "d4/nalgebra_lu_solve", + "d5/faer_lu_solve", + "d5/la_stack_lu_solve", + "d5/nalgebra_lu_solve", + "d64/faer_lu_solve", + "d64/la_stack_lu_solve", + "d64/nalgebra_lu_solve", + "d8/faer_lu_solve", + "d8/la_stack_lu_solve", + "d8/nalgebra_lu_solve", +] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "lu-solve" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "lu-solve", + "list", +] +timeout = 14400 + +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "lu-solve" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ + "d16/faer_lu_solve", + "d16/la_stack_lu_solve", + "d16/nalgebra_lu_solve", + "d2/faer_lu_solve", + "d2/la_stack_lu_solve", + "d2/nalgebra_lu_solve", + "d3/faer_lu_solve", + "d3/la_stack_lu_solve", + "d3/nalgebra_lu_solve", + "d32/faer_lu_solve", + "d32/la_stack_lu_solve", + "d32/nalgebra_lu_solve", + "d4/faer_lu_solve", + "d4/la_stack_lu_solve", + "d4/nalgebra_lu_solve", + "d5/faer_lu_solve", + "d5/la_stack_lu_solve", + "d5/nalgebra_lu_solve", + "d64/faer_lu_solve", + "d64/la_stack_lu_solve", + "d64/nalgebra_lu_solve", + "d8/faer_lu_solve", + "d8/la_stack_lu_solve", + "d8/nalgebra_lu_solve", +] + +[[suites.measurement.phases.current.discovery.suites]] +name = "lu-solve" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "lu-solve", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" +phase = "baseline" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "nalgebra" +phase = "current" + +[suites.measurement.series.rows] +"d2/la_stack_lu_solve" = "d2/nalgebra_lu_solve" +"d3/la_stack_lu_solve" = "d3/nalgebra_lu_solve" +"d4/la_stack_lu_solve" = "d4/nalgebra_lu_solve" +"d5/la_stack_lu_solve" = "d5/nalgebra_lu_solve" +"d8/la_stack_lu_solve" = "d8/nalgebra_lu_solve" +"d16/la_stack_lu_solve" = "d16/nalgebra_lu_solve" +"d32/la_stack_lu_solve" = "d32/nalgebra_lu_solve" +"d64/la_stack_lu_solve" = "d64/nalgebra_lu_solve" + +[[suites.measurement.series]] +name = "faer" +phase = "current" + +[suites.measurement.series.rows] +"d2/la_stack_lu_solve" = "d2/faer_lu_solve" +"d3/la_stack_lu_solve" = "d3/faer_lu_solve" +"d4/la_stack_lu_solve" = "d4/faer_lu_solve" +"d5/la_stack_lu_solve" = "d5/faer_lu_solve" +"d8/la_stack_lu_solve" = "d8/faer_lu_solve" +"d16/la_stack_lu_solve" = "d16/faer_lu_solve" +"d32/la_stack_lu_solve" = "d32/faer_lu_solve" +"d64/la_stack_lu_solve" = "d64/faer_lu_solve" + +[[suites]] +name = "column-basis" +contract = "la-stack-column-basis-v1" +since = "v0.5.2" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "column-basis" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "column-basis" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-column-basis" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ + "column_basis/2x2/full_rank", + "column_basis/2x2/large", + "column_basis/2x2/near_dependent", + "column_basis/2x2/rank_one", + "column_basis/2x2/small", + "column_basis/2x3/shared_face", + "column_basis/3x3/full_rank", + "column_basis/3x3/large", + "column_basis/3x3/near_dependent", + "column_basis/3x3/rank_one", + "column_basis/3x3/small", + "column_basis/3x4/shared_face", + "column_basis/4x4/full_rank", + "column_basis/4x4/large", + "column_basis/4x4/near_dependent", + "column_basis/4x4/rank_one", + "column_basis/4x4/small", + "column_basis/4x5/shared_face", + "column_basis/4x6/full_rank", + "column_basis/4x6/large", + "column_basis/4x6/near_dependent", + "column_basis/4x6/rank_one", + "column_basis/4x6/small", + "column_basis/5x2/full_rank", + "column_basis/5x2/large", + "column_basis/5x2/near_dependent", + "column_basis/5x2/rank_one", + "column_basis/5x2/small", + "column_basis/5x5/full_rank", + "column_basis/5x5/large", + "column_basis/5x5/near_dependent", + "column_basis/5x5/rank_one", + "column_basis/5x5/small", + "column_basis/5x6/shared_face", + "column_basis/5x8/full_rank", + "column_basis/5x8/large", + "column_basis/5x8/near_dependent", + "column_basis/5x8/rank_one", + "column_basis/5x8/small", + "column_basis/6x10/full_rank", + "column_basis/6x10/large", + "column_basis/6x10/near_dependent", + "column_basis/6x10/rank_one", + "column_basis/6x10/small", + "column_basis/6x7/shared_face", + "column_basis/7x12/full_rank", + "column_basis/7x12/large", + "column_basis/7x12/near_dependent", + "column_basis/7x12/rank_one", + "column_basis/7x12/small", + "column_basis/8x14/full_rank", + "column_basis/8x14/large", + "column_basis/8x14/near_dependent", + "column_basis/8x14/rank_one", + "column_basis/8x14/small", +] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "column-basis" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "column-basis", + "list", +] +timeout = 14400 + +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "column-basis" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-column-basis-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ + "column_basis/2x2/full_rank", + "column_basis/2x2/large", + "column_basis/2x2/near_dependent", + "column_basis/2x2/rank_one", + "column_basis/2x2/small", + "column_basis/2x3/shared_face", + "column_basis/3x3/full_rank", + "column_basis/3x3/large", + "column_basis/3x3/near_dependent", + "column_basis/3x3/rank_one", + "column_basis/3x3/small", + "column_basis/3x4/shared_face", + "column_basis/4x4/full_rank", + "column_basis/4x4/large", + "column_basis/4x4/near_dependent", + "column_basis/4x4/rank_one", + "column_basis/4x4/small", + "column_basis/4x5/shared_face", + "column_basis/4x6/full_rank", + "column_basis/4x6/large", + "column_basis/4x6/near_dependent", + "column_basis/4x6/rank_one", + "column_basis/4x6/small", + "column_basis/5x2/full_rank", + "column_basis/5x2/large", + "column_basis/5x2/near_dependent", + "column_basis/5x2/rank_one", + "column_basis/5x2/small", + "column_basis/5x5/full_rank", + "column_basis/5x5/large", + "column_basis/5x5/near_dependent", + "column_basis/5x5/rank_one", + "column_basis/5x5/small", + "column_basis/5x6/shared_face", + "column_basis/5x8/full_rank", + "column_basis/5x8/large", + "column_basis/5x8/near_dependent", + "column_basis/5x8/rank_one", + "column_basis/5x8/small", + "column_basis/6x10/full_rank", + "column_basis/6x10/large", + "column_basis/6x10/near_dependent", + "column_basis/6x10/rank_one", + "column_basis/6x10/small", + "column_basis/6x7/shared_face", + "column_basis/7x12/full_rank", + "column_basis/7x12/large", + "column_basis/7x12/near_dependent", + "column_basis/7x12/rank_one", + "column_basis/7x12/small", + "column_basis/8x14/full_rank", + "column_basis/8x14/large", + "column_basis/8x14/near_dependent", + "column_basis/8x14/rank_one", + "column_basis/8x14/small", +] + +[[suites.measurement.phases.current.discovery.suites]] +name = "column-basis" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "column-basis", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" +phase = "baseline" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites]] +name = "interval-solve" +contract = "la-stack-interval-solve-v1" +since = "v0.5.2" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "interval-solve" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "interval-solve" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-interval-solve" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ + "interval_division/negative", + "interval_division/positive", + "interval_division/reciprocal", + "interval_division/subnormal", + "interval_division/tiny_ratio", + "interval_solve_d1/dense", + "interval_solve_d1/near_singular", + "interval_solve_d1/range_exhausted", + "interval_solve_d1/singular", + "interval_solve_d1/uncertain_dense", + "interval_solve_d2/dense", + "interval_solve_d2/near_singular", + "interval_solve_d2/range_exhausted", + "interval_solve_d2/singular", + "interval_solve_d2/uncertain_dense", + "interval_solve_d3/dense", + "interval_solve_d3/near_singular", + "interval_solve_d3/range_exhausted", + "interval_solve_d3/singular", + "interval_solve_d3/uncertain_dense", + "interval_solve_d4/dense", + "interval_solve_d4/near_singular", + "interval_solve_d4/range_exhausted", + "interval_solve_d4/singular", + "interval_solve_d4/uncertain_dense", + "interval_solve_d5/dense", + "interval_solve_d5/near_singular", + "interval_solve_d5/range_exhausted", + "interval_solve_d5/singular", + "interval_solve_d5/uncertain_dense", + "interval_solve_d6/dense", + "interval_solve_d6/near_singular", + "interval_solve_d6/range_exhausted", + "interval_solve_d6/singular", + "interval_solve_d6/uncertain_dense", +] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "interval-solve" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "interval-solve", + "list", +] +timeout = 14400 + +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "interval-solve" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-interval-solve-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ + "interval_division/negative", + "interval_division/positive", + "interval_division/reciprocal", + "interval_division/subnormal", + "interval_division/tiny_ratio", + "interval_solve_d1/dense", + "interval_solve_d1/near_singular", + "interval_solve_d1/range_exhausted", + "interval_solve_d1/singular", + "interval_solve_d1/uncertain_dense", + "interval_solve_d2/dense", + "interval_solve_d2/near_singular", + "interval_solve_d2/range_exhausted", + "interval_solve_d2/singular", + "interval_solve_d2/uncertain_dense", + "interval_solve_d3/dense", + "interval_solve_d3/near_singular", + "interval_solve_d3/range_exhausted", + "interval_solve_d3/singular", + "interval_solve_d3/uncertain_dense", + "interval_solve_d4/dense", + "interval_solve_d4/near_singular", + "interval_solve_d4/range_exhausted", + "interval_solve_d4/singular", + "interval_solve_d4/uncertain_dense", + "interval_solve_d5/dense", + "interval_solve_d5/near_singular", + "interval_solve_d5/range_exhausted", + "interval_solve_d5/singular", + "interval_solve_d5/uncertain_dense", + "interval_solve_d6/dense", + "interval_solve_d6/near_singular", + "interval_solve_d6/range_exhausted", + "interval_solve_d6/singular", + "interval_solve_d6/uncertain_dense", +] + +[[suites.measurement.phases.current.discovery.suites]] +name = "interval-solve" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "interval-solve", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" +phase = "baseline" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites]] +name = "polynomial" +contract = "la-stack-polynomial-v1" +since = "v0.5.2" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "polynomial" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "polynomial" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-polynomial" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ "polynomial/coincident_lifted/order4/determinant", "polynomial/coincident_lifted/order5/determinant", "polynomial/coincident_lifted/order6/determinant", "polynomial/coincident_lifted/order7/determinant", "polynomial/coincident_lifted/order8/determinant" ] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "polynomial" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "polynomial", + "list", +] timeout = 14400 -[[phases.current.discovery.suites]] -name = "public-exact" -command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "current-public-exact", "list" ] +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "polynomial" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-polynomial-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ "polynomial/coincident_lifted/order4/determinant", "polynomial/coincident_lifted/order5/determinant", "polynomial/coincident_lifted/order6/determinant", "polynomial/coincident_lifted/order7/determinant", "polynomial/coincident_lifted/order8/determinant" ] + +[[suites.measurement.phases.current.discovery.suites]] +name = "polynomial" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "polynomial", + "list", +] timeout = 14400 -[[series]] +[[suites.measurement.series]] name = "la-stack baseline" phase = "baseline" -select = { include = [ "*" ], exclude = [ "d*/nalgebra_*", "d*/faer_*" ] } -[[series]] +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] name = "la-stack current" phase = "current" -select = { include = [ "*" ], exclude = [ "d*/nalgebra_*", "d*/faer_*" ] } -[[series]] -name = "nalgebra" -phase = "baseline" -[series.rows] -"d2/la_stack_lu_solve" = "d2/nalgebra_lu_solve" -"d3/la_stack_lu_solve" = "d3/nalgebra_lu_solve" -"d4/la_stack_lu_solve" = "d4/nalgebra_lu_solve" -"d5/la_stack_lu_solve" = "d5/nalgebra_lu_solve" -"d8/la_stack_lu_solve" = "d8/nalgebra_lu_solve" -"d16/la_stack_lu_solve" = "d16/nalgebra_lu_solve" -"d32/la_stack_lu_solve" = "d32/nalgebra_lu_solve" -"d64/la_stack_lu_solve" = "d64/nalgebra_lu_solve" +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] -[[series]] -name = "faer" +[[suites]] +name = "rational-dot" +contract = "la-stack-rational-dot-v1" +since = "v0.5.2" +nominal-seconds-per-case = 8 + +[suites.measurement] +schema = 4 +minimum-release = "v0.5.0" +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "dependency-sources.toml", + "rust-toolchain.toml", + "pyproject.toml", + "uv.lock", + ".python-version", + "tests/exact_bench_config.rs", + "tests/column_basis.rs", + "tests/rational_dot.rs", + "tests/interval_solve.rs", + "tests/common/interval_solve_oracle.rs", + "tests/common/proptest_config.rs", + "tests/polynomial_bench_inputs.rs", + "tooling/performance.just", + "tooling/performance-*.toml", + "tooling/*.py", + "tooling/criterion-inventory.toml", + "tooling/performance-policy.toml", + "tooling/measurements/*.toml", +] +sample-count = 100 +statistics = [ "mean", "median" ] +confidence-level = 0.95 +unit = "ns" +timeout = 21600 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] +backend = "export" +harness-mode = "common" +workloads = [ "benches/**/*.rs" ] + +[suites.measurement.context] +phase-driver = "just-v1" +suite = "rational-dot" +scope = "public-release-and-paired-lu-peers-v4" +repository = "acgetchell/la-stack" +acquisition-owner = "local" + +[suites.measurement.probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[suites.measurement.dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[suites.measurement.phases] + +[suites.measurement.phases.baseline] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "rational-dot" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-rational-dot" ] + +[suites.measurement.phases.baseline.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.baseline.discovery] +required = [ + "rational_dot/cancellation/d14/prepared", + "rational_dot/cancellation/d14/slice", + "rational_dot/cancellation/d2/prepared", + "rational_dot/cancellation/d2/slice", + "rational_dot/cancellation/d22/prepared", + "rational_dot/cancellation/d22/slice", + "rational_dot/cancellation/d3/prepared", + "rational_dot/cancellation/d3/slice", + "rational_dot/cancellation/d4/prepared", + "rational_dot/cancellation/d4/slice", + "rational_dot/cancellation/d5/prepared", + "rational_dot/cancellation/d5/slice", + "rational_dot/cancellation/d6/prepared", + "rational_dot/cancellation/d6/slice", + "rational_dot/cancellation/d7/prepared", + "rational_dot/cancellation/d7/slice", + "rational_dot/cancellation/d8/prepared", + "rational_dot/cancellation/d8/slice", + "rational_dot/large/d14/prepared", + "rational_dot/large/d14/slice", + "rational_dot/large/d2/prepared", + "rational_dot/large/d2/slice", + "rational_dot/large/d22/prepared", + "rational_dot/large/d22/slice", + "rational_dot/large/d3/prepared", + "rational_dot/large/d3/slice", + "rational_dot/large/d4/prepared", + "rational_dot/large/d4/slice", + "rational_dot/large/d5/prepared", + "rational_dot/large/d5/slice", + "rational_dot/large/d6/prepared", + "rational_dot/large/d6/slice", + "rational_dot/large/d7/prepared", + "rational_dot/large/d7/slice", + "rational_dot/large/d8/prepared", + "rational_dot/large/d8/slice", + "rational_dot/non_dyadic/d14/prepared", + "rational_dot/non_dyadic/d14/slice", + "rational_dot/non_dyadic/d2/prepared", + "rational_dot/non_dyadic/d2/slice", + "rational_dot/non_dyadic/d22/prepared", + "rational_dot/non_dyadic/d22/slice", + "rational_dot/non_dyadic/d3/prepared", + "rational_dot/non_dyadic/d3/slice", + "rational_dot/non_dyadic/d4/prepared", + "rational_dot/non_dyadic/d4/slice", + "rational_dot/non_dyadic/d5/prepared", + "rational_dot/non_dyadic/d5/slice", + "rational_dot/non_dyadic/d6/prepared", + "rational_dot/non_dyadic/d6/slice", + "rational_dot/non_dyadic/d7/prepared", + "rational_dot/non_dyadic/d7/slice", + "rational_dot/non_dyadic/d8/prepared", + "rational_dot/non_dyadic/d8/slice", +] + +[[suites.measurement.phases.baseline.discovery.suites]] +name = "rational-dot" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "rational-dot", + "list", +] +timeout = 14400 + +[suites.measurement.phases.current] +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "rational-dot" ] +gate = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "gate-rational-dot-current" ] + +[suites.measurement.phases.current.environment] +CARGO_TARGET_DIR = "target/build" +CRITERION_HOME = "target/criterion" +RUSTUP_TOOLCHAIN = "1.99.1" +RUSTFLAGS = "--cap-lints=warn" +CARGO_ENCODED_RUSTFLAGS = "--cap-lints=warn" +RUSTC = "rustc" +RUSTC_WRAPPER = "" +RUSTC_WORKSPACE_WRAPPER = "" +CARGO_BUILD_BUILD_DIR = "target/build/intermediates" + +[suites.measurement.phases.current.discovery] +required = [ + "rational_dot/cancellation/d14/prepared", + "rational_dot/cancellation/d14/slice", + "rational_dot/cancellation/d2/prepared", + "rational_dot/cancellation/d2/slice", + "rational_dot/cancellation/d22/prepared", + "rational_dot/cancellation/d22/slice", + "rational_dot/cancellation/d3/prepared", + "rational_dot/cancellation/d3/slice", + "rational_dot/cancellation/d4/prepared", + "rational_dot/cancellation/d4/slice", + "rational_dot/cancellation/d5/prepared", + "rational_dot/cancellation/d5/slice", + "rational_dot/cancellation/d6/prepared", + "rational_dot/cancellation/d6/slice", + "rational_dot/cancellation/d7/prepared", + "rational_dot/cancellation/d7/slice", + "rational_dot/cancellation/d8/prepared", + "rational_dot/cancellation/d8/slice", + "rational_dot/large/d14/prepared", + "rational_dot/large/d14/slice", + "rational_dot/large/d2/prepared", + "rational_dot/large/d2/slice", + "rational_dot/large/d22/prepared", + "rational_dot/large/d22/slice", + "rational_dot/large/d3/prepared", + "rational_dot/large/d3/slice", + "rational_dot/large/d4/prepared", + "rational_dot/large/d4/slice", + "rational_dot/large/d5/prepared", + "rational_dot/large/d5/slice", + "rational_dot/large/d6/prepared", + "rational_dot/large/d6/slice", + "rational_dot/large/d7/prepared", + "rational_dot/large/d7/slice", + "rational_dot/large/d8/prepared", + "rational_dot/large/d8/slice", + "rational_dot/non_dyadic/d14/prepared", + "rational_dot/non_dyadic/d14/slice", + "rational_dot/non_dyadic/d2/prepared", + "rational_dot/non_dyadic/d2/slice", + "rational_dot/non_dyadic/d22/prepared", + "rational_dot/non_dyadic/d22/slice", + "rational_dot/non_dyadic/d3/prepared", + "rational_dot/non_dyadic/d3/slice", + "rational_dot/non_dyadic/d4/prepared", + "rational_dot/non_dyadic/d4/slice", + "rational_dot/non_dyadic/d5/prepared", + "rational_dot/non_dyadic/d5/slice", + "rational_dot/non_dyadic/d6/prepared", + "rational_dot/non_dyadic/d6/slice", + "rational_dot/non_dyadic/d7/prepared", + "rational_dot/non_dyadic/d7/slice", + "rational_dot/non_dyadic/d8/prepared", + "rational_dot/non_dyadic/d8/slice", +] + +[[suites.measurement.phases.current.discovery.suites]] +name = "rational-dot" +command = [ + "just", + "--justfile", + "tooling/performance.just", + "--working-directory", + ".", + "rational-dot", + "list", +] +timeout = 14400 + +[[suites.measurement.series]] +name = "la-stack baseline" phase = "baseline" -[series.rows] -"d2/la_stack_lu_solve" = "d2/faer_lu_solve" -"d3/la_stack_lu_solve" = "d3/faer_lu_solve" -"d4/la_stack_lu_solve" = "d4/faer_lu_solve" -"d5/la_stack_lu_solve" = "d5/faer_lu_solve" -"d8/la_stack_lu_solve" = "d8/faer_lu_solve" -"d16/la_stack_lu_solve" = "d16/faer_lu_solve" -"d32/la_stack_lu_solve" = "d32/faer_lu_solve" -"d64/la_stack_lu_solve" = "d64/faer_lu_solve" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] + +[[suites.measurement.series]] +name = "la-stack current" +phase = "current" + +[suites.measurement.series.select] +include = [ "*" ] +exclude = [ "d*/nalgebra_*", "d*/faer_*" ] diff --git a/tooling/measurements/non-exact.toml b/tooling/measurements/non-exact.toml index 858dfd0..f72a6fe 100644 --- a/tooling/measurements/non-exact.toml +++ b/tooling/measurements/non-exact.toml @@ -1,13 +1,14 @@ # Scientific measurement policy consumed directly by research-repo-tools. schema = 2 minimum-release = "v0.5.0" -sources = [ "src/**/*.rs", "Cargo.toml" ] +sources = [ "src/**/*.rs", "Cargo.toml", "README.md" ] harness = [ "benches/**/*.rs", "benches/comparison/Cargo.toml", "examples/**/*.rs", "Cargo.toml", "Cargo.lock", + "dependency-sources.toml", "rust-toolchain.toml", "tests/exact_bench_config.rs", "tooling/performance.just", diff --git a/tooling/performance-policy.toml b/tooling/performance-policy.toml index cc1fc7e..4d50f60 100644 --- a/tooling/performance-policy.toml +++ b/tooling/performance-policy.toml @@ -75,10 +75,13 @@ title = "Matrix infinity norm (max abs row sum)" benchmarks = [ "la_stack_inf_norm", "nalgebra_inf_norm", "faer_inf_norm" ] # Independent release case catalog. Native discovery must agree with these IDs. -# These APIs were introduced in v0.5.2 and have no v0.5.0/v0.5.1 timings. -# Keep their complete current coverage; shared cases retain both revisions. -[baseline] -current-only = [ "column_basis/*", "interval_division/*", "interval_solve_d*/*", "polynomial/*", "rational_dot/*" ] +# Independently assert introduction versions on both sides of every comparison. +[availability] +"column_basis/*" = "v0.5.2" +"interval_division/*" = "v0.5.2" +"interval_solve_d*/*" = "v0.5.2" +"polynomial/*" = "v0.5.2" +"rational_dot/*" = "v0.5.2" [release] default = [ diff --git a/tooling/performance-readme.toml b/tooling/performance-readme.toml index 9e9d864..bcffb69 100644 --- a/tooling/performance-readme.toml +++ b/tooling/performance-readme.toml @@ -1,5 +1,6 @@ # Canonical README view; plot-only alternatives live under tooling/plots/. schema = 2 +trial = 1 document = "README.md" begin = "" end = "" @@ -23,11 +24,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_median_log.svg" @@ -57,10 +58,12 @@ source = "version" [[references]] path = "docs/performance.md" -pattern = '^- la-stack baseline: measured in \*\*baseline\*\* phase \((?Pv[0-9]+\.[0-9]+\.[0-9]+)\),' +pattern = '^- baseline: (?:un)?available [^\r\n]*; release (?Pv[0-9]+\.[0-9]+\.[0-9]+); revision\b' source = "previous-tag" +count = 7 [[references]] path = "docs/performance.md" -pattern = '^- la-stack current: measured in \*\*current\*\* phase \((?Pv[0-9]+\.[0-9]+\.[0-9]+)\),' +pattern = '^- current: (?:un)?available [^\r\n]*; release (?Pv[0-9]+\.[0-9]+\.[0-9]+); revision\b' source = "tag" +count = 7 diff --git a/tooling/performance.just b/tooling/performance.just index 0912117..1e4a629 100644 --- a/tooling/performance.just +++ b/tooling/performance.just @@ -8,44 +8,81 @@ export CRITERION_HOME := absolute_path(env("CRITERION_HOME", "target/criterion") # Compare native targets on the measured host regardless of a caller's target override. host_target := `rustc --print host-tuple` -# Each public workload has one owner per phase. Peers are acquired once locally. -baseline-all mode="measure": budget (baseline-public-default mode) (baseline-public-exact mode) (baseline-peers mode) - -baseline-peers mode="measure": (peers mode) - -# The baseline predates v0.5.2's new APIs; retain comparisons for shared workloads. -baseline-public-default mode="measure": (common-public-default mode) - -baseline-public-exact mode="measure": (common-public-exact mode) +# Discovery and integration gates must both compile without the generated README. +[private] +[script("uv", "run", "--locked", "--no-sync", "python", "-I")] +_readme-input-check: + import re + from pathlib import Path + + root = Path("target/build").resolve(strict=True) + profiles = set() + files = sorted(root.rglob("la_stack-*.d")) + if not files: + raise SystemExit("missing la-stack compiler dependency files") + for path in files: + if not re.fullmatch(r"la_stack-[0-9a-f]+\.d", path.name): + raise SystemExit(f"unexpected la-stack dependency filename: {path}") + if path.is_symlink() or path.resolve(strict=True) != path: + raise SystemExit(f"aliased compiler dependency file: {path}") + data = path.read_bytes().lower().replace(b"\\", b"/") + if b"src/lib.rs" not in data: + raise SystemExit(f"missing library input in compiler dependency file: {path}") + if b"readme.md" in data: + raise SystemExit("current compilation reads README.md; full acquisition requires documentation-only inclusion; use non-exact for historical current tags") + profiles.update(set(path.relative_to(root).parts) & {"debug", "release"}) + if profiles != {"debug", "release"}: + raise SystemExit("missing benchmark-discovery or correctness-gate compiler dependencies") baseline-vs_linalg mode="measure": cargo bench --locked --target {{ host_target }} -p la-stack-comparison --features bench --bench vs_linalg -- --noplot {{ if mode == "list" { "--list" } else { "" } }} -# The independent case catalog describes the complete local acquisition budget. -budget: - uv run --locked --no-sync python -c 'import fnmatch, pathlib, tomllib; policy = tomllib.loads(pathlib.Path("tooling/performance-policy.toml").read_text()); current = sum(len(policy["release"][name]) for name in ("default", "exact")); baseline = sum(not any(fnmatch.fnmatchcase(case, pattern) for pattern in policy["baseline"]["current-only"]) for name in ("default", "exact") for case in policy["release"][name]); total = baseline + current + 16; print(f"Local release: {baseline} baseline + {current} current cases, 16 peer cases once; {total} measurements, {total * 8 / 60:.1f} nominal minutes at 3 s warmup + 5 s measurement (excludes compilation, gates and analysis).")' +column-basis mode="measure": (native "column_basis" "" "bench" mode) -common-public-default mode="measure" sample="": (native "vs_linalg" '^d[0-9]+/la_stack_' "bench" mode sample) (native "angle" '/(prepared_vector|borrowed_slice|construct_then_angle)$' "bench" mode sample) (native "gram" '/la_stack$' "bench" mode sample) (native "interval" "" "bench" mode sample) (native "linear_form" "" "bench" mode sample) (native "public_api" "" "bench" mode sample) +common-public-default mode="measure" sample="": (native "vs_linalg" '^d[0-9]+/la_stack_' "bench" mode sample) (default-operations mode sample) common-public-exact mode="measure" sample="": (native "exact" '^(canonical_conversion_|exact_(d[2-5]|random_corpus_d[2-5]|near_singular_3x3|large_entries_3x3|hilbert_[45]x[45])/|rational_input_.*d[2-8]/(det_sign_row_cleared_bareiss|det_row_cleared_bareiss|solve_row_cleared_bareiss)$)' "bench,exact" mode sample) (native "rational_solve" '/(construct|prepared|adapter)$' "bench,exact" mode sample) (native "public_api" '^public_exact_' "bench,exact" mode sample) -current-all mode="measure": (current-public-default mode) (current-public-exact mode) - -current-public-default mode="measure": (public-default mode) - -current-public-exact mode="measure": (public-exact mode) +# LU factor-and-solve and its peers have their own paired acquisition unit. +core-default mode="measure": (native "vs_linalg" '^d[0-9]+/la_stack_(det.*|dot|inf_norm.*|ldlt.*|lu|lu_pivoting|lu_ill_conditioned|norm2.*|solve.*)$' "bench" mode) (default-operations mode) current-vs_linalg mode="measure": cargo bench --locked --target {{ host_target }} -p la-stack-comparison --features bench --bench vs_linalg -- la_stack --noplot {{ if mode == "list" { "--list" } else { "" } }} +default-operations mode="measure" sample="": (native "angle" '/(prepared_vector|borrowed_slice|construct_then_angle)$' "bench" mode sample) (native "gram" '/la_stack$' "bench" mode sample) (native "interval" "" "bench" mode sample) (native "linear_form" "" "bench" mode sample) (native "public_api" "" "bench" mode sample) + gate: cargo test --locked --target {{ host_target }} --workspace --features bench,exact --test vs_linalg_inputs --test exact_bench_config -gate-public: gate - cargo test --locked --target {{ host_target }} --features bench,exact --test column_basis --test rational_dot --test interval_solve --test polynomial_bench_inputs +gate-column-basis: + cargo test --locked --target {{ host_target }} --features bench --test column_basis + +gate-column-basis-current: gate-column-basis _readme-input-check + +gate-current: gate _readme-input-check + +gate-interval-solve: + cargo test --locked --target {{ host_target }} --features bench,exact --test interval_solve + +gate-interval-solve-current: gate-interval-solve _readme-input-check + +gate-polynomial: + cargo test --locked --target {{ host_target }} --features bench,exact --test polynomial_bench_inputs + +gate-polynomial-current: gate-polynomial _readme-input-check + +gate-public: gate gate-column-basis gate-interval-solve gate-polynomial gate-rational-dot + +gate-rational-dot: + cargo test --locked --target {{ host_target }} --features bench,exact --test rational_dot + +gate-rational-dot-current: gate-rational-dot _readme-input-check +interval-solve mode="measure": (native "interval_solve" "" "bench" mode) la-stack mode="measure" sample="": (public-default mode sample) (public-exact mode sample) +lu-solve mode="measure": (native "vs_linalg" '^d(2|3|4|5|8|16|32|64)/(la_stack|nalgebra|faer)_lu_solve$' "bench" mode) + # Shared native argument construction keeps discovery and measurement identical. native suite filter features mode="measure" sample="": #!/usr/bin/env bash @@ -64,6 +101,10 @@ native suite filter features mode="measure" sample="": peers mode="measure" sample="": (native "vs_linalg" '^d(2|3|4|5|8|16|32|64)/(nalgebra|faer)_lu_solve$' "bench" mode sample) +polynomial mode="measure": (native "polynomial" "" "bench,exact" mode) + public-default mode="measure" sample="": (common-public-default mode sample) (native "column_basis" "" "bench" mode sample) (native "interval_solve" "" "bench" mode sample) public-exact mode="measure" sample="": (common-public-exact mode sample) (native "polynomial" "" "bench,exact" mode sample) (native "rational_dot" "" "bench,exact" mode sample) + +rational-dot mode="measure": (native "rational_dot" "" "bench,exact" mode) diff --git a/tooling/plots/retained/lu_solve-mean-linear.toml b/tooling/plots/retained/lu_solve-mean-linear.toml index c5ab421..220f6a9 100644 --- a/tooling/plots/retained/lu_solve-mean-linear.toml +++ b/tooling/plots/retained/lu_solve-mean-linear.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" archive = "docs/performance-runs" statistic = "mean" @@ -17,11 +18,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_mean_linear.svg" diff --git a/tooling/plots/retained/lu_solve-mean-log.toml b/tooling/plots/retained/lu_solve-mean-log.toml index 8bf8416..b20cecc 100644 --- a/tooling/plots/retained/lu_solve-mean-log.toml +++ b/tooling/plots/retained/lu_solve-mean-log.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" archive = "docs/performance-runs" statistic = "mean" @@ -17,11 +18,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_mean_log.svg" diff --git a/tooling/plots/retained/lu_solve-median-linear.toml b/tooling/plots/retained/lu_solve-median-linear.toml index d1a2aec..1f9dba9 100644 --- a/tooling/plots/retained/lu_solve-median-linear.toml +++ b/tooling/plots/retained/lu_solve-median-linear.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" archive = "docs/performance-runs" statistic = "median" @@ -17,11 +18,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_median_linear.svg" diff --git a/tooling/plots/retained/lu_solve-median-log.toml b/tooling/plots/retained/lu_solve-median-log.toml index 06a8ede..671fc52 100644 --- a/tooling/plots/retained/lu_solve-median-log.toml +++ b/tooling/plots/retained/lu_solve-median-log.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" archive = "docs/performance-runs" statistic = "median" @@ -17,11 +18,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_median_log.svg" diff --git a/tooling/plots/scratch/lu_solve-mean-linear.toml b/tooling/plots/scratch/lu_solve-mean-linear.toml index bd68839..a0d626d 100644 --- a/tooling/plots/scratch/lu_solve-mean-linear.toml +++ b/tooling/plots/scratch/lu_solve-mean-linear.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" payload = "target/bench-reports/full/run.json" manifest = "target/bench-reports/full/evidence.json" @@ -18,11 +19,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_mean_linear.svg" diff --git a/tooling/plots/scratch/lu_solve-mean-log.toml b/tooling/plots/scratch/lu_solve-mean-log.toml index a2814bd..ff4e73e 100644 --- a/tooling/plots/scratch/lu_solve-mean-log.toml +++ b/tooling/plots/scratch/lu_solve-mean-log.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" payload = "target/bench-reports/full/run.json" manifest = "target/bench-reports/full/evidence.json" @@ -18,11 +19,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_mean_log.svg" diff --git a/tooling/plots/scratch/lu_solve-median-linear.toml b/tooling/plots/scratch/lu_solve-median-linear.toml index b76f745..24bbb05 100644 --- a/tooling/plots/scratch/lu_solve-median-linear.toml +++ b/tooling/plots/scratch/lu_solve-median-linear.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" payload = "target/bench-reports/full/run.json" manifest = "target/bench-reports/full/evidence.json" @@ -18,11 +19,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_median_linear.svg" diff --git a/tooling/plots/scratch/lu_solve-median-log.toml b/tooling/plots/scratch/lu_solve-median-log.toml index c1c50cb..79aac28 100644 --- a/tooling/plots/scratch/lu_solve-median-log.toml +++ b/tooling/plots/scratch/lu_solve-median-log.toml @@ -1,5 +1,6 @@ # Named view consumed directly by `performance publish --artifacts-only`. schema = 2 +trial = 1 measurement = "tooling/measurements/full.toml" payload = "target/bench-reports/full/run.json" manifest = "target/bench-reports/full/evidence.json" @@ -18,11 +19,11 @@ rows = [ { key = "64", label = "64", coordinate = 64 }, ] series = [ - { name = "la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, - { name = "nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, - { name = "faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, + { name = "lu-solve/la-stack current", label = "la-stack", label-version = "release", rows = { "2" = "d2/la_stack_lu_solve", "3" = "d3/la_stack_lu_solve", "4" = "d4/la_stack_lu_solve", "5" = "d5/la_stack_lu_solve", "8" = "d8/la_stack_lu_solve", "16" = "d16/la_stack_lu_solve", "32" = "d32/la_stack_lu_solve", "64" = "d64/la_stack_lu_solve" } }, + { name = "lu-solve/nalgebra", label = "nalgebra", label-version = "dependency.nalgebra", rows = { "2" = "d2/nalgebra_lu_solve", "3" = "d3/nalgebra_lu_solve", "4" = "d4/nalgebra_lu_solve", "5" = "d5/nalgebra_lu_solve", "8" = "d8/nalgebra_lu_solve", "16" = "d16/nalgebra_lu_solve", "32" = "d32/nalgebra_lu_solve", "64" = "d64/nalgebra_lu_solve" } }, + { name = "lu-solve/faer", label = "faer", label-version = "dependency.faer", rows = { "2" = "d2/faer_lu_solve", "3" = "d3/faer_lu_solve", "4" = "d4/faer_lu_solve", "5" = "d5/faer_lu_solve", "8" = "d8/faer_lu_solve", "16" = "d16/faer_lu_solve", "32" = "d32/faer_lu_solve", "64" = "d64/faer_lu_solve" } }, ] -comparisons = [ { numerator = "nalgebra", denominator = "la-stack current" }, { numerator = "faer", denominator = "la-stack current" } ] +comparisons = [ { numerator = "lu-solve/nalgebra", denominator = "lu-solve/la-stack current" }, { numerator = "lu-solve/faer", denominator = "lu-solve/la-stack current" } ] [plot] path = "docs/assets/bench/vs_linalg_lu_solve_median_log.svg" diff --git a/tooling/release_candidate.py b/tooling/release_candidate.py new file mode 100644 index 0000000..cef126a --- /dev/null +++ b/tooling/release_candidate.py @@ -0,0 +1,574 @@ +"""Retain local acquisition assertions separately from the release candidate.""" + +import argparse +import hashlib +import json +import os +import re +import tempfile +from dataclasses import dataclass, replace +from functools import partial +from pathlib import Path + +from research_repo_tools.candidate_publication import plan_candidate_publication +from research_repo_tools.candidate_reuse import ( + CandidatePolicy, + acquisition_assertion_digest, + parse_candidate_policy, + plan_candidate_archive, + serialize_candidate_policy, + verify_candidate_archive, + verify_candidate_release, +) +from research_repo_tools.cli import main as shared_main +from research_repo_tools.common_measurement import validate_run_policy +from research_repo_tools.complete_runs import run_identity +from research_repo_tools.config import load as load_config +from research_repo_tools.dependency_provenance import check_release_dependencies +from research_repo_tools.evidence import parse_evidence +from research_repo_tools.files import replace_many +from research_repo_tools.measurement import load_measurement, resolve_revision +from research_repo_tools.performance_progress import ProgressReporter +from research_repo_tools.performance_release import load_performance_release, load_preparation, measure_release, serialize_preparation +from research_repo_tools.publication import publish_publication +from research_repo_tools.release_assets import ( + ReleaseTarget, + download_release_asset, + parse_release_target, + preflight_release_target, + publish_release_asset, + revalidate_release_target, + serialize_release_target, +) +from research_repo_tools.release_interfaces import load_evidence_policy +from research_repo_tools.release_publishing import ReviewedRelease, check_reviewed_release +from research_repo_tools.release_validation import api +from research_repo_tools.scheduled_runs import ScheduledRun +from research_repo_tools.suite_runs import parse_suite_run + +SETTINGS = "tooling/performance-release.toml" +REPOSITORY = "acgetchell/la-stack" +ROLES = ("policy", "publication", "acceptance", "archive") + + +def owner_directory(root: Path, checkpoint: str, selected: Path | None = None) -> Path: + """Keep the operator's immutable record outside candidate-controlled files.""" + identity = hashlib.sha256(str((root / checkpoint).resolve()).encode("utf-8")).hexdigest() + directory = (selected or Path.home() / ".local/state/la-stack/acquisitions" / identity).expanduser().resolve() + if directory.is_relative_to(root) or root.is_relative_to(directory): + message = "acquisition owner directory must be separate from the checkout" + raise ValueError(message) + return directory + + +def freeze(directory: Path, files: dict[str, bytes]) -> None: + """Atomically retain owner assertions; retries cannot replace earlier bytes.""" + directory = directory.resolve(strict=True) + updates = {directory / name: data for name, data in files.items()} + if any(not path.resolve().is_relative_to(directory) for path in updates): + message = "owner files must remain within their record directory" + raise ValueError(message) + + def unchanged() -> None: + for path, data in updates.items(): + if path.exists() and path.read_bytes() != data: + raise ValueError(f"immutable output differs: {path.name}") + + unchanged() + replace_many(updates, verify=unchanged) + + +def capture(root: Path, owner: Path, *, retained_scratch: bool = False) -> dict[str, str]: + """Finish the trusted owner's receipt from its already retained original bytes.""" + settings = load_performance_release(root, SETTINGS) + preparation = load_preparation(root, settings) + if serialize_preparation(preparation) != (owner / "preparation.json").read_bytes(): + message = "independent preparation changed since acquisition" + raise ValueError(message) + if retained_scratch: + # Explicit owner recovery after a failed callback; never infer this + # authority from an incoming artifact or silently fall back to scratch. + preparation.verify(root) + freeze(owner, {"run.json": (root / settings.payload).read_bytes(), "evidence.json": (root / settings.manifest).read_bytes()}) + payload, manifest = (owner / "run.json").read_bytes(), (owner / "evidence.json").read_bytes() + # Restore only identical/missing scratch evidence for a failed capture retry. + freeze(root, {settings.payload: payload, settings.manifest: manifest}) + evidence = parse_evidence(payload, manifest) + run = validate_run_policy(evidence, load_measurement(root, settings.measurement), release=True) + if not isinstance(run, ScheduledRun): + message = "candidate acquisition requires the declared trial schedule" + raise TypeError(message) + policy = CandidatePolicy( + settings.repository, + preparation.pair.current, + None, + "local", + run_identity(evidence), + run.plan, + dict(evidence.sources)["current"], + tuple( + (index, suite, phase, source) for index, trial in enumerate(run.trials, 1) for suite, phase, _sample, source in parse_suite_run(trial).observations + ), + report=settings.report, + ) + acquisition = acquisition_assertion_digest(policy) + freeze(owner, {"policy.json": serialize_candidate_policy(policy), "acquisition.sha256": (acquisition + "\n").encode("ascii")}) + receipt_path = owner / "publication.json" + if not receipt_path.exists(): + _plan, receipt = plan_candidate_publication(root, settings.payload, settings.manifest, policy, acquisition_sha256=acquisition) + freeze(owner, {"publication.json": receipt}) + publication = hashlib.sha256(receipt_path.read_bytes()).hexdigest() + freeze(owner, {"publication.sha256": (publication + "\n").encode("ascii")}) + return {"owner_directory": str(owner), "acquisition_sha256": acquisition, "publication_sha256": publication} + + +def acquire(root: Path, owner: Path, checkpoint: str, *, resume: bool, progress: ProgressReporter) -> dict[str, str]: + """Acquire under owner control, freezing assertions before report promotion.""" + settings = load_performance_release(root, SETTINGS) + preparation = load_preparation(root, settings) + check_release_dependencies(root) + if resume: + if not owner.is_dir(): + message = "resume requires the original acquisition owner record" + raise ValueError(message) + else: + try: + owner.mkdir(mode=0o700, parents=True) + except FileExistsError as error: + if not owner.exists(): + raise + message = ( + f"acquisition owner record already exists: {owner}; preserve it and select a new checkpoint/owner directory, " + "or use performance-resume if the original checkpoint still exists" + ) + raise ValueError(message) from error + freeze(owner, {"preparation.json": serialize_preparation(preparation)}) + + def before_promotion() -> None: + freeze(owner, {"run.json": (root / settings.payload).read_bytes(), "evidence.json": (root / settings.manifest).read_bytes()}) + capture(root, owner) + + measure_release( + root, + settings, + allow_git_mutations=True, + checkpoint=checkpoint, + resume=resume, + progress=progress, + before_promotion=before_promotion, + release_intent=True, + ) + return { + "owner_directory": str(owner), + "acquisition_sha256": (owner / "acquisition.sha256").read_text(encoding="ascii").strip(), + "publication_sha256": (owner / "publication.sha256").read_text(encoding="ascii").strip(), + } + + +def final_directory(owner: Path, commit: str, selection: str) -> Path: + """Name an explicit immutable review slot; never replace an earlier selection.""" + if re.fullmatch(r"[0-9a-f]{40}", commit) is None or re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", selection) is None: + message = "final evidence requires a full commit and a short lowercase review selection name" + raise ValueError(message) + return owner / "final" / commit / selection + + +def verify(root: Path, owner: Path, selection: str = "reviewed") -> Path: + """Bind the separately trusted record to clean HEAD and signed native CI.""" + check_release_dependencies(root) + commit = resolve_revision(root, "HEAD") + policy = replace(parse_candidate_policy((owner / "policy.json").read_bytes()), commit=commit) + final = final_directory(owner, commit, selection) + final.mkdir(parents=True, exist_ok=True) + freeze(final, {"policy.json": serialize_candidate_policy(policy)}) + accepted = final / "acceptance.json" + with tempfile.TemporaryDirectory(prefix="la-stack-candidate-verification-") as temporary: + output = Path(temporary).resolve() / "acceptance.json" + arguments = [ + "--root", + str(root), + "release", + "evidence", + "candidate", + "tooling/release-evidence.toml", + "--checkout", + str(root), + "--policy", + str(final / "policy.json"), + "--payload", + str(owner / "run.json"), + "--manifest", + str(owner / "evidence.json"), + "--publication-receipt", + str(owner / "publication.json"), + "--acquisition-sha256", + (owner / "acquisition.sha256").read_text(encoding="ascii").strip(), + "--publication-sha256", + (owner / "publication.sha256").read_text(encoding="ascii").strip(), + "--output", + str(output), + "--format", + "summary", + ] + if accepted.exists(): + arguments.extend(("--acceptance", str(accepted))) + status = shared_main(arguments) + if status: + raise RuntimeError(f"candidate verification failed with exit {status}") + freeze(final, {"acceptance.json": output.read_bytes()}) + return accepted + + +def archive(owner: Path, commit: str, selection: str = "reviewed") -> Path: + """Derive an immutable release archive without acquisition or native work.""" + final = final_directory(owner, commit, selection) + policy = parse_candidate_policy((final / "policy.json").read_bytes()) + if policy.commit != commit: + message = "selected candidate policy differs from the independently selected commit" + raise ValueError(message) + prefix = final.relative_to(owner).as_posix() + output = f"{prefix}/la-stack-{policy.tag}-criterion-baseline.tar.gz" + publish_publication(plan_candidate_archive(owner, "run.json", "evidence.json", f"{prefix}/policy.json", f"{prefix}/acceptance.json", output)) + verify_candidate_archive(owner / output, policy, (final / "acceptance.json").read_bytes()) + return owner / output + + +@dataclass(frozen=True, slots=True) +class Transfer: + """A parsed owner selection; only an authorized invocation authenticates it.""" + + target: ReleaseTarget + acquisition: str + publication: str + assets: tuple[tuple[str, str, int, str], ...] + + +def transfer_asset(role: str, raw: object) -> tuple[str, str, int, str]: + """Parse one exact inert asset selection.""" + if not isinstance(raw, dict) or set(raw) != {"name", "id", "sha256"}: + message = "invalid candidate asset assertion" + raise ValueError(message) + name, identifier, digest = raw["name"], raw["id"], raw["sha256"] + if not isinstance(name, str) or re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}", name) is None or type(identifier) is not int or identifier <= 0: + message = "candidate assets require safe names and positive exact IDs" + raise ValueError(message) + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + message = "candidate asset requires a full SHA-256 assertion" + raise ValueError(message) + return role, name, identifier, digest + + +def parse_transfer(payload: bytes) -> Transfer: + """Reject malformed or ambiguous operator assertions before provider access.""" + + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + if len(dict(pairs)) != len(pairs): + message = "duplicate transfer field" + raise ValueError(message) + return dict(pairs) + + data = json.loads(payload, object_pairs_hook=unique) + if ( + not isinstance(data, dict) + or set(data) != {"schema", "target", "acquisition", "publication", "assets"} + or type(data["schema"]) is not int + or data["schema"] != 1 + ): + message = "invalid candidate transfer schema" + raise ValueError(message) + target = parse_release_target(json.dumps(data["target"]).encode("utf-8")) + if target.repository != REPOSITORY or target.expected_title != target.tag: + message = "candidate transfer must select this repository and its exact release title" + raise ValueError(message) + + def digest(value: object) -> str: + if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: + message = "candidate transfer requires full SHA-256 assertions" + raise ValueError(message) + return value + + raw_assets = data["assets"] + if not isinstance(raw_assets, dict) or set(raw_assets) not in (set(ROLES), {*ROLES, "attached"}): + message = "candidate transfer requires four original assets and an optional attached archive" + raise ValueError(message) + assets = tuple(transfer_asset(role, raw_assets[role]) for role in (*ROLES, *(["attached"] if "attached" in raw_assets else []))) + if len({item[1] for item in assets}) != len(assets) or len({item[2] for item in assets}) != len(assets): + message = "candidate assets must have distinct names and IDs" + raise ValueError(message) + if "attached" in raw_assets and (raw_assets["attached"]["name"] != f"la-stack-{target.tag}-criterion-baseline.tar.gz" or assets[-1][3] != assets[3][3]): + message = "canonical attached archive must match the originally accepted bytes" + raise ValueError(message) + return Transfer(target, digest(data["acquisition"]), digest(data["publication"]), assets) + + +def stage(root: Path, owner: Path, selection: str) -> Path: + """Upload inert original records, retaining dispatch assertions independently.""" + accepted = verify(root, owner, selection) + commit = resolve_revision(root, "HEAD") + final = final_directory(owner, commit, selection) + policy = parse_candidate_policy((final / "policy.json").read_bytes()) + artifact = archive(owner, commit, selection) + target_path = final / "target.json" + if not target_path.exists(): + target = preflight_release_target(root, REPOSITORY, policy.tag, commit, expected_title=policy.tag, checkout=root) + freeze(final, {"target.json": serialize_release_target(target)}) + target = parse_release_target(target_path.read_bytes()) + if (target.repository, target.tag, target.commit) != (REPOSITORY, policy.tag, commit): + message = "staging target differs from the accepted candidate" + raise ValueError(message) + revalidate_release_target(root, target, checkout=root) + inputs = {"policy": final / "policy.json", "publication": owner / "publication.json", "acceptance": accepted, "archive": artifact} + transport = final / "transport" + transport.mkdir(exist_ok=True) + assertions: dict[str, dict[str, str | int]] = {} + for role, source in inputs.items(): + suffix = "tar.gz" if role == "archive" else "json" + name = f"la-stack-{policy.tag}-{selection}-candidate-{role}.{suffix}" + payload = source.read_bytes() + freeze(transport, {name: payload}) + publish_release_asset(root, REPOSITORY, policy.tag, transport / name, target=target, checkout=root) + release = revalidate_release_target(root, target, checkout=root) + entry = next(item for item in release.assets if item[0] == name) + assertions[role] = {"name": name, "id": entry[1], "sha256": hashlib.sha256(payload).hexdigest()} + payload = ( + json.dumps( + { + "schema": 1, + "target": json.loads(serialize_release_target(target)), + "acquisition": (owner / "acquisition.sha256").read_text(encoding="ascii").strip(), + "publication": (owner / "publication.sha256").read_text(encoding="ascii").strip(), + "assets": assertions, + }, + sort_keys=True, + ).encode("utf-8") + + b"\n" + ) + parse_transfer(payload) + freeze(final, {"transfer.json": payload}) + return final / "transfer.json" + + +def seal(root: Path, owner: Path, selection: str) -> Path: + """Independently select the canonical attached asset before publication.""" + final = final_directory(owner, resolve_revision(root, "HEAD"), selection) + payload = (final / "transfer.json").read_bytes() + transfer = parse_transfer(payload) + release = revalidate_release_target(root, transfer.target, checkout=root) + name = f"la-stack-{transfer.target.tag}-criterion-baseline.tar.gz" + entry = next((item for item in release.assets if item[0] == name), None) + if entry is None: + message = "canonical accepted benchmark archive is not yet attached" + raise ValueError(message) + digest = next(item[3] for item in transfer.assets if item[0] == "archive") + with tempfile.TemporaryDirectory(prefix="la-stack-seal-") as temporary: + download_release_asset(root, REPOSITORY, transfer.target.tag, name, Path(temporary) / "archive.tar.gz", expected_sha256=digest, allow_draft=True) + if entry not in revalidate_release_target(root, transfer.target, checkout=root).assets: + message = "attached archive identity changed during verification" + raise ValueError(message) + data = json.loads(payload) + data["assets"]["attached"] = {"name": name, "id": entry[1], "sha256": digest} + sealed = json.dumps(data, sort_keys=True).encode("utf-8") + b"\n" + parse_transfer(sealed) + freeze(final, {"publication-transfer.json": sealed}) + return final / "publication-transfer.json" + + +def require_dispatch(transfer: Transfer, *, published: bool) -> None: + """Bind independent workflow input to the allowed operator, workflow and tag.""" + workflow = "publish-crates.yml" if published else "release-benchmarks.yml" + expected = { + "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_REF": f"refs/tags/{transfer.target.tag}", + "GITHUB_SHA": transfer.target.commit, + "GITHUB_WORKFLOW_REF": f"{REPOSITORY}/.github/workflows/{workflow}@refs/tags/{transfer.target.tag}", + "GITHUB_ACTOR": "acgetchell", + "GITHUB_TRIGGERING_ACTOR": "acgetchell", + "RELEASE_TAG": transfer.target.tag, + } + if any(os.environ.get(key) != value for key, value in expected.items()): + message = "candidate handoff requires the maintainer's exact workflow/tag dispatch" + raise ValueError(message) + + +def transfer_files(root: Path, transfer: Transfer, directory: Path, incoming: Path | None) -> dict[str, Path]: + """Read exact dispatch-selected bytes from published assets or inert transport.""" + files: dict[str, Path] = {} + for role, name, _identifier, digest in transfer.assets: + path = directory / name + if incoming is None: + download_release_asset(root, REPOSITORY, transfer.target.tag, name, path, expected_sha256=digest) + else: + source = incoming / ("archive.tar.gz" if role == "archive" else f"{role}.json") + data = source.read_bytes() + if source.is_symlink() or hashlib.sha256(data).hexdigest() != digest: + message = "transported candidate bytes differ from independent dispatch assertions" + raise ValueError(message) + freeze(directory, {name: data}) + files[role] = path + return files + + +def check_published_transfer(root: Path, transfer: Transfer) -> None: + """Require the immutable release and exact original and canonical assets.""" + release = check_reviewed_release( + load_config(root=root), + transfer.target.tag, + event=ReviewedRelease(transfer.target.tag, transfer.target.commit, transfer.target.release_id), + ) + if not release.immutable: + message = "published candidate release must be immutable" + raise ValueError(message) + if (release.repository, release.tag, release.identifier, release.title) != ( + REPOSITORY, + transfer.target.tag, + transfer.target.release_id, + transfer.target.expected_title, + ): + message = "candidate release identity changed" + raise ValueError(message) + for _role, name, identifier, digest in transfer.assets: + entry = next((item for item in release.assets if item[0] == name), None) + if entry is None or entry[1] != identifier or (entry[3] is not None and entry[3] != digest): + message = "candidate asset identity changed" + raise ValueError(message) + + +@dataclass(frozen=True, slots=True) +class ReviewedAcceptance: + """Independent owner digest, optionally paired with the frozen approval bytes.""" + + sha256: str + expected: bytes | None = None + + def __post_init__(self) -> None: + """Reject missing review identity or a different approval receipt.""" + if re.fullmatch(r"[0-9a-f]{64}", self.sha256) is None: + message = "publication requires the independently reviewed full acceptance SHA-256" + raise ValueError(message) + if self.expected is not None and hashlib.sha256(self.expected).hexdigest() != self.sha256: + message = "approval receipt differs from independently reviewed acceptance" + raise ValueError(message) + + +def receive(root: Path, destination: Path, payload: bytes, *, source: Path | ReviewedAcceptance) -> Path: + """Read-only hosted acceptance, pinned to original bytes and reviewed CI.""" + published = isinstance(source, ReviewedAcceptance) + transfer = parse_transfer(payload) + require_dispatch(transfer, published=published) + if published and "attached" not in {item[0] for item in transfer.assets}: + message = "publication requires the independently selected canonical archive" + raise ValueError(message) + check_release_dependencies(root) + destination = owner_directory(root, "", destination) + destination.mkdir(parents=True, exist_ok=True) + + def release_identity() -> None: + if published: + check_published_transfer(root, transfer) + + release_identity() + with tempfile.TemporaryDirectory(prefix="la-stack-transfer-") as temporary: + files = transfer_files(root, transfer, Path(temporary), source if isinstance(source, Path) else None) + release_identity() + policy = parse_candidate_policy(files["policy"].read_bytes()) + accepted = files["acceptance"].read_bytes() + evidence = verify_candidate_archive(files["archive"], policy, accepted, target=transfer.target) + actual = verify_candidate_release( + root, + evidence, + policy, + load_evidence_policy(root / "tooling/release-evidence.toml"), + partial(api, repository=REPOSITORY), + acquisition_sha256=transfer.acquisition, + publication_receipt=files["publication"].read_bytes(), + publication_sha256=transfer.publication, + # A published release may need explicitly reviewed newer native CI. + # Before approval, the independent owner digest authenticates the + # newly verified result. After approval, pin those exact bytes. + expected=source.expected if isinstance(source, ReviewedAcceptance) else accepted, + ) + if isinstance(source, ReviewedAcceptance) and hashlib.sha256(actual).hexdigest() != source.sha256: + message = "verified acceptance differs from the independently reviewed digest" + raise ValueError(message) + release_identity() + name = f"la-stack-{transfer.target.tag}-criterion-baseline.tar.gz" + freeze( + destination, + { + "transfer.json": payload, + "acceptance.json": actual, + "target.json": serialize_release_target(transfer.target), + name: files["archive"].read_bytes(), + }, + ) + return destination / name + + +def dispatch_source(options: argparse.Namespace) -> Path | ReviewedAcceptance: + """Parse mutually exclusive draft transport and published approval inputs.""" + if options.published: + if options.incoming is not None: + message = "published verification requires live immutable assets" + raise ValueError(message) + return ReviewedAcceptance(os.environ.get("CANDIDATE_ACCEPTANCE", ""), options.expected.read_bytes() if options.expected is not None else None) + if options.incoming is None or options.expected is not None: + message = "draft verification requires inert files and no publication approval override" + raise ValueError(message) + return Path(options.incoming) + + +def main() -> None: + """Expose only repository orchestration; shared APIs own every evidence check.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + measurement = commands.add_parser("acquire") + measurement.add_argument("--checkpoint", required=True) + measurement.add_argument("--resume", action="store_true") + measurement.add_argument("--owner-directory", type=Path) + measurement.add_argument("--progress", choices=("human", "json", "none"), default="human") + measurement.add_argument("--progress-cadence", type=float, default=300) + for name in ("capture", "verify", "archive", "stage", "seal"): + command = commands.add_parser(name) + command.add_argument("owner_directory", type=Path) + if name == "capture": + command.add_argument("--retained-scratch", action="store_true", help="explicitly retry retention of the original locally controlled acquisition") + else: + command.add_argument("--selection", default="reviewed", help="immutable review record; choose a new name to review a different native CI attempt") + receiver = commands.add_parser("receive") + receiver.add_argument("destination", type=Path) + receiver.add_argument("--published", action="store_true") + receiver.add_argument("--incoming", type=Path) + receiver.add_argument("--expected", type=Path) + options = parser.parse_args() + root = Path.cwd().resolve() + if options.operation == "acquire": + owner = owner_directory(root, options.checkpoint, options.owner_directory) + print(f"Independent acquisition record: {owner}", flush=True) + result = acquire( + root, + owner, + options.checkpoint, + resume=options.resume, + progress=ProgressReporter(mode=options.progress, cadence=options.progress_cadence), + ) + print(json.dumps(result)) + elif options.operation == "receive": + print(receive(root, options.destination, os.environ["CANDIDATE_TRANSFER"].encode("utf-8"), source=dispatch_source(options))) + else: + owner = owner_directory(root, "", options.owner_directory) + if options.operation == "capture": + print(json.dumps(capture(root, owner, retained_scratch=options.retained_scratch))) + elif options.operation == "verify": + accepted = verify(root, owner, options.selection) + print(json.dumps({"acceptance": str(accepted), "sha256": hashlib.sha256(accepted.read_bytes()).hexdigest()})) + elif options.operation == "archive": + print(archive(owner, resolve_revision(root, "HEAD"), options.selection)) + elif options.operation == "stage": + print(stage(root, owner, options.selection)) + else: + print(seal(root, owner, options.selection)) + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index f8ede4f..68e378d 100644 --- a/uv.lock +++ b/uv.lock @@ -271,6 +271,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453, upload-time = "2024-07-12T22:25:58.476Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "face" version = "26.0.1" @@ -408,6 +417,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ { name = "actionlint-py" }, + { name = "pytest-xdist" }, { name = "research-repo-tools", extra = ["python-tools", "semgrep"] }, { name = "shellcheck-py" }, { name = "shfmt-py" }, @@ -422,12 +432,13 @@ tooling = [ [package.metadata.requires-dev] dev = [ { name = "actionlint-py", specifier = "==1.7.12.25" }, - { name = "research-repo-tools", extras = ["python-tools", "semgrep"], specifier = "==0.2.6" }, + { name = "pytest-xdist", specifier = "==3.8.0" }, + { name = "research-repo-tools", extras = ["python-tools", "semgrep"], git = "https://github.com/acgetchell/research-repo-tools?rev=055353c630604b1ae3438c1eaf534ba85e067a7e" }, { name = "shellcheck-py", specifier = "==0.11.0.1" }, { name = "shfmt-py", specifier = "==4.2.0" }, { name = "yamllint", specifier = "==1.38.0" }, ] -tooling = [{ name = "research-repo-tools", extras = ["python-tools", "semgrep"], specifier = "==0.2.6" }] +tooling = [{ name = "research-repo-tools", extras = ["python-tools", "semgrep"], git = "https://github.com/acgetchell/research-repo-tools?rev=055353c630604b1ae3438c1eaf534ba85e067a7e" }] [[package]] name = "markdown-it-py" @@ -811,6 +822,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.4" @@ -876,16 +900,12 @@ wheels = [ [[package]] name = "research-repo-tools" version = "0.2.6" -source = { registry = "https://pypi.org/simple" } +source = { git = "https://github.com/acgetchell/research-repo-tools?rev=055353c630604b1ae3438c1eaf534ba85e067a7e#055353c630604b1ae3438c1eaf534ba85e067a7e" } dependencies = [ { name = "packaging" }, { name = "pyyaml" }, { name = "rust-just" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/96/73185e1070954e856cc2ca3a99141bbc17e487950717fcff5859587c6f7b/research_repo_tools-0.2.6.tar.gz", hash = "sha256:dd826a916f037f47b925ca3268adc76d415a8561547bedda348a67f4f2e2e065", size = 1209707, upload-time = "2026-11-05T15:57:52.368Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/8e/e712f3e57bc1141f2dd89624aeb106a99200c3bc7611c7ce1686eb0a5fb3/research_repo_tools-0.2.6-py3-none-any.whl", hash = "sha256:89ebae476b5bde0cffc321603ed1d867796e3276a7fb88c7dc1f7de53a13e7c2", size = 496754, upload-time = "2026-11-05T15:57:50.732Z" }, -] [package.optional-dependencies] python-tools = [ @@ -1137,15 +1157,15 @@ wheels = [ [[package]] name = "sse-starlette" -version = "3.6.0" +version = "3.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "starlette" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/7f/3b8ec4f27833f7b8c270a3ef960b6d435e1065439847bc0fcd1e58ae898b/sse_starlette-3.6.0.tar.gz", hash = "sha256:623151d6c20dc0eebf1b3379c27faae91fff883bac7ab6665f16f46440a88b3a", size = 41174, upload-time = "2026-10-25T20:31:59.372Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/43/a232e3cd804293bd4f6d01e8895bc0f6f83ea148d712baf9d783c7d93d23/sse_starlette-3.7.0.tar.gz", hash = "sha256:326416da2ab52e9a41b3f409d3ff897eda45aa68c9f3b7b17d0913aab8d219c3", size = 43869, upload-time = "2026-11-07T11:15:49.389Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/49/35053078cdfcf639237e8bee6dc6f34f1b3a3cc8d01800a1e213e1660ff6/sse_starlette-3.6.0-py3-none-any.whl", hash = "sha256:e4dfa4d237d2712cb2b4d4918eda9b457cd6b44bbecdc838af75de836565bafd", size = 17985, upload-time = "2026-10-25T20:31:58.134Z" }, + { url = "https://files.pythonhosted.org/packages/6a/11/25cabac0a24fa76f58637d8671b549c90d818d69d26beecf0c59a3b186e0/sse_starlette-3.7.0-py3-none-any.whl", hash = "sha256:f1d3136caa12d129284fca3d2ded79bc9af943320b6472712955fea52a6d3eea", size = 18432, upload-time = "2026-11-07T11:15:47.86Z" }, ] [[package]]