diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d447e14..29c3294 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,17 +8,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true -# NOT the org's sharded-tests reusable, deliberately. That one shards a package's -# suite and expects TestShards in the test environment; this repository's library -# has one trivial test and would be forcing a dependency on everyone who presses -# "Use this template". What actually needs checking here is different anyway: that -# a project environment RESOLVES and that its scripts run — a green library beside -# a project that cannot instantiate would be the wrong kind of green. +# NOT the org's sharded-tests reusable, deliberately. That one shards a package's suite and expects +# TestShards in the test environment, which would force a dependency on everyone who presses "Use +# this template". What needs checking here is different: that every project environment resolves and +# that its smoke sweep runs. A green library beside a project that cannot instantiate would be the +# wrong kind of green. # -# `setup.sh` is deliberately NOT run here. It rewrites names and re-issues UUIDs, -# which is a step the person starting a study should take knowingly rather than -# find already done for them. The cost is that its failure modes are not covered -# by CI, so a change to it has to be exercised by hand. +# `setup.sh` is deliberately NOT run here. It rewrites names and re-issues UUIDs, which is a step the +# person starting a study should take knowingly rather than find already done for them. The cost is +# that its failure modes are not covered by CI, so a change to it has to be exercised by hand. jobs: library: runs-on: ubuntu-latest @@ -30,20 +28,62 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + # Every directory under `projects/` that carries a Project.toml is its own environment, so each + # gets its own run rooted there. Discovered from the tree rather than listed here: a list and a + # tree disagree the first time someone copies a project and forgets to add it. + discover: + runs-on: ubuntu-latest + outputs: + projects: ${{ steps.find.outputs.projects }} + steps: + - uses: actions/checkout@v7 + - id: find + run: | + set -euo pipefail + found="$(find projects -mindepth 2 -maxdepth 2 -name Project.toml -printf '%h\n' | sort)" + test -n "$found" || { echo "no projects/*/Project.toml found" >&2; exit 1; } + echo "$found" | sed 's/^/ /' >&2 + echo "projects=$(printf '%s' "$found" | jq -R . | jq -sc .)" >> "$GITHUB_OUTPUT" + project: + needs: discover + strategy: + fail-fast: false + matrix: + project: ${{ fromJson(needs.discover.outputs.projects) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 with: { version: '1' } - uses: julia-actions/cache@v3 - - name: Instantiate the example project - working-directory: projects/ExampleSweep + - name: Instantiate + working-directory: ${{ matrix.project }} run: julia --project=. -e 'using Pkg; Pkg.instantiate()' - name: Its own tests - working-directory: projects/ExampleSweep - run: julia --project=. test/runtests.jl - - name: The reduction both faces call - working-directory: projects/ExampleSweep - run: julia --project=. scripts/collect.jl out + working-directory: ${{ matrix.project }} + run: | + set -euo pipefail + test -f test/runtests.jl || { echo "no test/runtests.jl" >&2; exit 1; } + julia --project=. test/runtests.jl + # Every project ships `configs/smoke.toml`: the size that proves the wiring before a queue + # does. Running it here is what makes this job mean "the sweep works", not "it resolved". + - name: Smoke sweep, then read it back + working-directory: ${{ matrix.project }} + run: | + set -euo pipefail + test -f configs/smoke.toml || { echo "no configs/smoke.toml" >&2; exit 1; } + julia --project=. scripts/compute.jl configs/smoke.toml + julia --project=. scripts/collect.jl configs/smoke.toml + # One stable context for the branch ruleset to require. A matrix job's own checks are named after + # the matrix entry, so requiring `project` directly would require a context that stops being + # reported the moment a project is added or renamed. + projects-passed: + needs: project + if: always() + runs-on: ubuntu-latest + steps: + - run: | + test "${{ needs.project.result }}" = "success" \ + || { echo "a project failed: ${{ needs.project.result }}" >&2; exit 1; } diff --git a/docs/make.jl b/docs/make.jl index ed25a7f..9d8ac21 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,7 +1,3 @@ using Documenter, MyModule -makedocs(; - sitename="MyModule", - modules=[MyModule], - pages=["Home" => "index.md"], -) +makedocs(; sitename="MyModule", modules=[MyModule], pages=["Home" => "index.md"]) diff --git a/projects/ExampleSweep/configs/debug.toml b/projects/ExampleSweep/configs/debug.toml index b3a8930..458de54 100644 --- a/projects/ExampleSweep/configs/debug.toml +++ b/projects/ExampleSweep/configs/debug.toml @@ -1,7 +1,25 @@ -# Small but not trivial — the size you submit once to see the queue behave. -[sweep] -L = [8, 12] -h = [0.5, 1.0, 1.5] +# One file, read by all three layers: +# ParamIO.load(this) -> ConfigSpec ([study], [datavault], [[paramsets]]) +# DataVault.Vault(this) -> Vault ([study] for project_name/outdir, +# [datavault] path_keys for directory names) +# SweepRunner.run! -> over the DataKeys ParamIO expands from it +# +# A LIST value is a swept axis; a SCALAR is fixed. In DataKey.params the entries +# appear under their DOTTED names, e.g. "system.a". -[run] -out = "out" +[study] +project_name = "example" +total_samples = 1 +outdir = "out" + +[datavault] +# Which params name the on-disk directory and the canonical key identity. +path_keys = ["system.a", "numerics.dt"] + +[[paramsets]] + +[paramsets.system] +a = [0.5, 1.0] + +[paramsets.numerics] +dt = [1.0e-2, 5.0e-3] diff --git a/projects/ExampleSweep/configs/production.toml b/projects/ExampleSweep/configs/production.toml index 6c4092d..95d4e80 100644 --- a/projects/ExampleSweep/configs/production.toml +++ b/projects/ExampleSweep/configs/production.toml @@ -1,8 +1,25 @@ -# The real sweep. Same file shape as `smoke.toml`; only the ranges differ, so a -# config that works small is the config that runs big. -[sweep] -L = [8, 12, 16, 20] -h = [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0] - -[run] -out = "out" +# One file, read by all three layers: +# ParamIO.load(this) -> ConfigSpec ([study], [datavault], [[paramsets]]) +# DataVault.Vault(this) -> Vault ([study] for project_name/outdir, +# [datavault] path_keys for directory names) +# SweepRunner.run! -> over the DataKeys ParamIO expands from it +# +# A LIST value is a swept axis; a SCALAR is fixed. In DataKey.params the entries +# appear under their DOTTED names, e.g. "system.a". + +[study] +project_name = "example" +total_samples = 3 +outdir = "out" + +[datavault] +# Which params name the on-disk directory and the canonical key identity. +path_keys = ["system.a", "numerics.dt"] + +[[paramsets]] + +[paramsets.system] +a = [0.25, 0.5, 1.0, 2.0, 4.0] + +[paramsets.numerics] +dt = [1.0e-2, 5.0e-3, 2.5e-3, 1.25e-3] diff --git a/projects/ExampleSweep/configs/smoke.toml b/projects/ExampleSweep/configs/smoke.toml index ebbd752..2bf5f4c 100644 --- a/projects/ExampleSweep/configs/smoke.toml +++ b/projects/ExampleSweep/configs/smoke.toml @@ -1,7 +1,25 @@ -# Two points. Runs in seconds on a laptop; proves the wiring before a queue does. -[sweep] -L = [8] -h = [0.5, 1.0] +# One file, read by all three layers: +# ParamIO.load(this) -> ConfigSpec ([study], [datavault], [[paramsets]]) +# DataVault.Vault(this) -> Vault ([study] for project_name/outdir, +# [datavault] path_keys for directory names) +# SweepRunner.run! -> over the DataKeys ParamIO expands from it +# +# A LIST value is a swept axis; a SCALAR is fixed. In DataKey.params the entries +# appear under their DOTTED names, e.g. "system.a". -[run] -out = "out" +[study] +project_name = "example" +total_samples = 1 +outdir = "out" + +[datavault] +# Which params name the on-disk directory and the canonical key identity. +path_keys = ["system.a", "numerics.dt"] + +[[paramsets]] + +[paramsets.system] +a = [1.0] + +[paramsets.numerics] +dt = [1.0e-2] diff --git a/projects/ExampleSweep/report/report.jl b/projects/ExampleSweep/report/report.jl index 9ecc8df..839178a 100644 --- a/projects/ExampleSweep/report/report.jl +++ b/projects/ExampleSweep/report/report.jl @@ -1,8 +1,17 @@ -# Draw from the finished vault. Run with `--project=report`, never the compute env. +# Draw from the finished vault. Run with `--project=report`, never the compute env — +# that is the whole reason the two environments are separate. # -# julia --project=report report/report.jl out -using DataVault, ExampleSweep +# julia --project=report report/report.jl configs/smoke.toml -vault = DataVault.Vault(get(ARGS, 1, "out")) -s = ExampleSweep.summarise(vault) # the same reduction scripts/collect.jl uses -@info "reporting over" s... +using DataVault: DataVault +using ExampleSweep: ExampleSweep +using ParamIO: ParamIO + +const CONFIG = get(ARGS, 1, joinpath(@__DIR__, "..", "configs", "smoke.toml")) +const OUTDIR = get(ENV, "DATAVAULT_OUTDIR", joinpath(@__DIR__, "..", "out")) + +vault = DataVault.Vault(CONFIG; run="phase1", outdir=OUTDIR) +rows = ExampleSweep.summarise(vault) # the same reduction scripts/collect.jl uses + +@info "reporting over" n = length(rows) +# Draw here — Pinax and a plotting backend are dependencies of THIS environment. diff --git a/projects/ExampleSweep/scripts/collect.jl b/projects/ExampleSweep/scripts/collect.jl index c27ebd9..34dbdc7 100644 --- a/projects/ExampleSweep/scripts/collect.jl +++ b/projects/ExampleSweep/scripts/collect.jl @@ -1,9 +1,22 @@ -# Read the finished vault and report it as text. No plotting dependency, so this -# can run on the machine that did the compute. +# The reader side. `compute.jl` never called save! — the runtime persisted every +# work_fn return for us. No plotting dependency, so this runs where the compute did. # -# julia --project=. scripts/collect.jl out -using DataVault, ExampleSweep +# julia --project=. scripts/collect.jl configs/smoke.toml -vault = DataVault.Vault(get(ARGS, 1, "out")) -s = ExampleSweep.summarise(vault) # the same reduction report/report.jl uses -@info "collected" s... +using DataVault: DataVault +using ExampleSweep: ExampleSweep +using ParamIO: ParamIO +using Printf + +const CONFIG = get(ARGS, 1, joinpath(@__DIR__, "..", "configs", "smoke.toml")) +const OUTDIR = get(ENV, "DATAVAULT_OUTDIR", joinpath(@__DIR__, "..", "out")) + +vault = DataVault.Vault(CONFIG; run="phase1", outdir=OUTDIR) +rows = ExampleSweep.summarise(vault) # the same reduction report/ uses + +@printf("\n %-8s %-10s %s\n", "a", "dt", "rel_error") +println(" ─────────────────────────────────────") +for r in rows + @printf(" %-8.4g %-10.4g %.3e\n", r.a, r.dt, r.rel_error) +end +@printf("\n %d points\n\n", length(rows)) diff --git a/projects/ExampleSweep/scripts/compute.jl b/projects/ExampleSweep/scripts/compute.jl index 23a7953..2e94778 100644 --- a/projects/ExampleSweep/scripts/compute.jl +++ b/projects/ExampleSweep/scripts/compute.jl @@ -1,20 +1,46 @@ -# One entry point for every size of run. `julia --project=. scripts/compute.jl configs/smoke.toml` -# -# What SweepRunner adds over a `for` loop: a point already finished is skipped -# after one manifest read, two processes pointed at the same vault never compute -# the same point twice, and a killed run is continued rather than restarted. - -using DataVault -using MyModule -using ParamIO -using SweepRunner - -config = get(ARGS, 1, "configs/smoke.toml") -spec = ParamIO.load(config) -vault = DataVault.Vault(spec["run"]["out"]) -keys = ParamIO.enumerate_keys(spec["sweep"]) - -work_fn(key) = MyModule.solve(; ParamIO.params(key)...) - -result = SweepRunner.run!(work_fn, vault, keys; load = MyModule) -SweepRunner.launchable(result) || exit(1) +#============================================================================== + compute.jl — the three-layer driver. This is the file to copy, not to invent. + + ParamIO : config TOML -> Vector{DataKey} (what to compute) + DataVault : (study, run) -> file storage (where it goes) + SweepRunner: run!(work_fn) -> parallel runtime (do it, lock-safe, resumable) + + The work itself is `MyModule.work_fn`, in a PACKAGE rather than in this script. + That is what lets `run!(…; load=MyModule)` hand it to the workers — no + `@everywhere`, and nothing to forget broadcasting. + + julia --project=. scripts/compute.jl configs/smoke.toml + + Run it again and it exits in milliseconds: the manifest records what is done. +==============================================================================# + +# `using X: X` brings in the MODULE and none of its exports, so every call below has to name where +# it comes from. That is a discipline the seam needs rather than a style choice: ClassicalMonteCarlo +# — a plausible work package for this slot — also exports `run!`, and a bare `using` of both would +# make `run!` ambiguous at the one call that matters. Qualifying keeps `SweepRunner.run!` the sweep's +# `run!` no matter what the work package is called or what it exports. +using DataVault: DataVault +using MyModule: MyModule +using ParamIO: ParamIO +using SweepRunner: SweepRunner + +const CONFIG = get(ARGS, 1, joinpath(@__DIR__, "..", "configs", "smoke.toml")) +# outdir precedence, resolved by DataVault: kwarg > ENV > the config's [study]. +const OUTDIR = get(ENV, "DATAVAULT_OUTDIR", joinpath(@__DIR__, "..", "out")) + +spec = ParamIO.load(CONFIG) +keys = ParamIO.expand(spec) +vault = DataVault.Vault(CONFIG; run="phase1", outdir=OUTDIR) + +SweepRunner.init_workers!(; mode=:auto) + +# work_fn RETURNS a Dict; the runtime saves it and writes the .done marker. +# batch/run.sh traps the wall-clock signal and touches this file, at which point +# run! stops dispatching new keys and returns cleanly instead of being killed. +opts = SweepRunner.RunOpts(; stop_flag=get(ENV, "PM_STOP_FLAG", nothing)) + +result = SweepRunner.run!(MyModule.work_fn, vault, keys; opts=opts, load=MyModule) +@info "phase1 complete" result + +ledger = DataVault.build_ledger(vault) # one row per completed key +@info "ledger written" ledger diff --git a/projects/ExampleSweep/src/ExampleSweep.jl b/projects/ExampleSweep/src/ExampleSweep.jl index 9a2390d..7a28781 100644 --- a/projects/ExampleSweep/src/ExampleSweep.jl +++ b/projects/ExampleSweep/src/ExampleSweep.jl @@ -1,24 +1,26 @@ module ExampleSweep -# Project-local code: how THIS study reduces its own results. -# -# It lives here, and not in `scripts/` or in `report/`, because both of those -# call it. A summary printed on the cluster and a figure drawn afterwards then -# cannot report different numbers — there is only one function to be wrong. +# This study's own reduction. It lives here, and not in `scripts/` or `report/`, +# because both call it — a summary printed on the cluster and a figure drawn +# afterwards then cannot report different numbers. -using DataVault +using DataVault: DataVault export summarise """ - summarise(vault) -> NamedTuple + summarise(vault) -> Vector{NamedTuple} -Reduce a finished vault to whatever this study reports. Replace the body; keep -the shape, so `scripts/collect.jl` and `report/report.jl` stay in agreement. +Read every finished point back and reduce it. Replace the body; keep the shape, +so `scripts/collect.jl` and `report/report.jl` stay in agreement. """ function summarise(vault) - ks = DataVault.keys(vault) - return (; n_points=length(ks)) + rows = NamedTuple[] + for key in DataVault.keys(vault; status=:done) + d = DataVault.load(vault, key) + push!(rows, (; a=d["a"], dt=d["dt"], rel_error=d["rel_error"])) + end + return sort(rows; by=r -> (r.a, r.dt)) end end # module ExampleSweep diff --git a/src/MyModule.jl b/src/MyModule.jl index ec3ba3a..92a9adb 100644 --- a/src/MyModule.jl +++ b/src/MyModule.jl @@ -1,16 +1,51 @@ +""" +MyModule — the study's own code, and the function the sweep calls. + +Intentionally DEPENDENCY-FREE, like the kernel in SweepRunner's own examples. +`run!(…; load=MyModule)` ships this package to the workers, and everything it +`using`s travels with it — so keeping it lean is what makes the fan-out cheap +and what removes the whole "module not defined on a worker" class of bug. + +The placeholder below is a damped exponential, integrated by explicit Euler. It +is here because it has an EXACT answer (`x(t) = exp(-a t)`), so `test/` can check +the result rather than only that something ran. Replace it with the real physics; +keep the shape of `work_fn`. +""" module MyModule -# Shared code. A project's `scripts/compute.jl` calls into here, so the same -# function is what runs on one core and on a thousand. +export decay_error, work_fn -export solve +""" + decay_error(a, dt; t_end=1.0) -> Float64 + +Relative error of explicit Euler on `x' = -a x`, `x(0) = 1`, at `t_end`, against +the exact `exp(-a t_end)`. First order in `dt`, which is the property the test +asserts: halving `dt` halves the error. +""" +function decay_error(a::Float64, dt::Float64; t_end::Float64=1.0)::Float64 + x = 1.0 + n = round(Int, t_end / dt) + for _ in 1:n + x += -a * x * dt + end + exact = exp(-a * t_end) + return abs(x - exact) / exact +end """ - solve(; kwargs...) + work_fn(key) -> Dict{String,Any} + +One parameter point. Reads the swept values off the `DataKey` by their DOTTED +names (`"group.leaf"`, matching the `[paramsets.group]` block in the config) and +RETURNS the payload. -Replace this with the study's actual computation. It takes the parameters of ONE -point and returns what should be stored for that point. +It does not save anything: the runtime calls `DataVault.save!` with whatever this +returns and writes the `.done` marker. Returning a non-`Dict` is a runtime error. """ -solve(; kwargs...) = error("MyModule.solve is not implemented yet") +function work_fn(key) + a = Float64(key.params["system.a"]) + dt = Float64(key.params["numerics.dt"]) + return Dict{String,Any}("a" => a, "dt" => dt, "rel_error" => decay_error(a, dt)) +end end # module MyModule diff --git a/test/runtests.jl b/test/runtests.jl index 4b6aae1..1057119 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,5 +2,16 @@ using MyModule using Test @testset "MyModule" begin - @test isdefined(MyModule, :solve) + # The placeholder kernel is first-order in dt, so halving dt halves the error. + # Asserting the RATE rather than a threshold is what makes this a real test: + # a guessed tolerance passes for the wrong reason as soon as the method changes. + e1 = MyModule.decay_error(1.0, 1e-2) + e2 = MyModule.decay_error(1.0, 5e-3) + @test 1.8 < e1 / e2 < 2.2 + + # …and the sweep's entry point returns what the runtime will store. + key = (; params=Dict("system.a" => 1.0, "numerics.dt" => 1e-2), sample=1) + d = MyModule.work_fn(key) + @test d isa Dict{String,Any} + @test haskey(d, "rel_error") end