diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36fd221..8610290 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,9 @@ jobs: # Excluded (require external services): # ✗ S3, GCS, Azure, MongoDB, Mnesia # - # Exclusions are configured in test/test_helper.exs + # Exclusions are configured in test/test_helper.exs. Wall-clock and + # memory benchmarks (:performance, :memory) are excluded there too and + # run in the non-blocking "Benchmarks" job below. # To run all tests locally: # mix test --include s3 --include gcs --include azure --include mongo --include mnesia run: mix test --trace @@ -182,6 +184,42 @@ jobs: echo "Checking for undocumented public functions..." mix docs --warnings-as-errors + # Wall-clock and memory benchmarks. Shared runners vary in speed and load, so + # these report regressions without blocking merges: a failure shows in this + # job's log and annotations while the workflow stays green. + benchmarks: + name: Benchmarks (non-blocking) + runs-on: ubuntu-latest + continue-on-error: true + env: + MIX_ENV: test + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19.5" + otp-version: "28.1" + + - name: Restore dependencies cache + uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-benchmarks-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix-benchmarks- + + - name: Install dependencies + run: mix deps.get + + - name: Run benchmark tests + run: mix test --only performance --only memory + livebooks: name: Livebooks runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e85b12..c11438f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,8 +85,9 @@ jobs: run: mix compile --warnings-as-errors # Functional suite. Wall-clock and memory benchmarks (:performance, - # :memory) are noisy on shared runners when tests run in parallel; CI - # runs them serially (mix test --trace) on every push to main. + # :memory) are excluded by test/test_helper.exs; the flags below repeat + # that so the release gate does not depend on it. CI runs them in its + # non-blocking "Benchmarks" job. - name: Run tests run: mix test --exclude performance --exclude memory diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4ce90..2a3cace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,12 @@ docs that match measured behaviour. No public API was removed or renamed. say that Explorer streaming was planned for 1.3 and dropped. ### Tests +- Wall-clock and memory benchmark tests (`:performance`, `:memory`) are + excluded from the default `mix test` run and from blocking CI. They failed + CI three times on shared runners with no code change (for example, an 8 MB + `from_tensor/2` took 6.3 s on one runner and 0.64 s locally). CI runs them + in a separate non-blocking "Benchmarks" job; run them locally with + `mix test --only performance --only memory`. - The streaming memory test measures the test process instead of the whole VM, so other async tests no longer move the result; the limit is now 2 MB. - New tests cover `stream_chunk_tensors/2` options (`:include_missing`, diff --git a/test/test_helper.exs b/test/test_helper.exs index c128f42..3ed2c21 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -13,6 +13,16 @@ # # To run live cloud/Python tests locally: # mix test --include s3 --include gcs --include azure --include mongo --include mnesia --include python -ExUnit.configure(exclude: [:s3, :gcs, :azure, :mongo, :mnesia, :python, :python_fixtures]) +# +# Wall-clock and memory benchmarks are also excluded by default: +# - :performance - asserts elapsed time (e.g. an 8 MB conversion under 5 s) +# - :memory - asserts memory growth while streaming +# Their results depend on how loaded the machine is, so on shared CI runners +# they fail without any code change. CI runs them in the non-blocking +# "Benchmarks" job. Run them locally with: +# mix test --only performance --only memory +ExUnit.configure( + exclude: [:s3, :gcs, :azure, :mongo, :mnesia, :python, :python_fixtures, :performance, :memory] +) ExUnit.start()