From ef7ef781018f27f910a03ebbe9d7597ae5d3506d Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Tue, 22 Sep 2026 11:21:55 +0000 Subject: [PATCH 1/4] Seams found end to end: face paths, cleanup, a lean site, v1 bindings (0.4.3) From running SweepRunner -> DataVault -> Pinax.report -> deposit on archeion-demo: - deposit takes the file Pinax.render/report returns (index.html, agent.json) as its directory. - a deposit that fails at any step, not only in provenance, leaves no _incoming/. - the built site leaves out provenance/ and repro/: at 79_500 points the table is 24.4 MiB per revision (4.5 MiB gzip; git packs a re-render at +0 and a full recompute at +4.3 MiB), and the site would carry it once per revision. provenance.toml is still served. - validate warns on a loaded-matches-disk from observation_version 1, whose check for code defined in a script never fired (DataVault 0.8.6). Co-Authored-By: Claude Opus 5 (1M context) --- Project.toml | 2 +- SPEC.md | 5 +++-- src/build.jl | 16 +++++++++++++++- src/deposit.jl | 35 ++++++++++++++++++++--------------- src/provenance.jl | 10 ++++++++++ test/test_deposit.jl | 31 +++++++++++++++++++++++++++++++ test/test_provenance.jl | 24 ++++++++++++++++++++++-- 7 files changed, 102 insertions(+), 21 deletions(-) diff --git a/Project.toml b/Project.toml index 87d4821..cc65aee 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Archeion" uuid = "4f7774e9-efb7-42d6-b735-122dcc7ea7b8" -version = "0.4.2" +version = "0.4.3" authors = ["sota shimozono "] [deps] diff --git a/SPEC.md b/SPEC.md index 7bc1626..12b0e4f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -186,8 +186,9 @@ key file read_sha256 result_sha256 observation completed_at - A depositor refuses a row whose `read_sha256` differs from its `result_sha256` unless told to let it in, which is then recorded as `allow_mismatch = true`. A validator requires the summary to match the table, every named token to be held or listed as missing, every binding to be one of the three - values, and every snapshot to hash to its id; it warns on rows that read other bytes and on - missing observations. + values, and every snapshot to hash to its id; it warns on rows that read other bytes, on + missing observations, and on a `loaded-matches-disk` from an observation of + `observation_version = 1`, whose check for code defined in a script never fired. - When `provenance.toml` exists, `repro/observations/`, `repro/sources/` and `repro/blobs/` are its own. diff --git a/src/build.jl b/src/build.jl index 7387b7c..e3414af 100644 --- a/src/build.jl +++ b/src/build.jl @@ -318,6 +318,20 @@ end # ── build ───────────────────────────────────────────────────────────────────────────────────── +# What of a revision the site serves: everything a reader opens, not the evidence behind it. The +# point table and `repro/` (observations, source snapshots and contents) stay in the repository: +# a table of 80_000 points is 24 MiB per revision, and a site that copied it would carry it once +# per revision ever deposited. The summary, `provenance.toml`, is served and links nowhere. +const SITE_SKIP = ("provenance", "repro") + +function copy_revision(src, dest) + mkpath(dest) + for name in readdir(src) + name in SITE_SKIP && isdir(joinpath(src, name)) && continue + cp(joinpath(src, name), joinpath(dest, name)) + end +end + function build(root, out=joinpath(root, "_site"); name=basename(abspath(root))) r, _ = validate(root) isempty(r.errors) || error( @@ -336,7 +350,7 @@ function build(root, out=joinpath(root, "_site"); name=basename(abspath(root))) dest = joinpath(out, rec.rel) mkpath(joinpath(dest, "revisions")) for rev in rec.revs - cp(rev.dir, joinpath(dest, "revisions", rev.name)) + copy_revision(rev.dir, joinpath(dest, "revisions", rev.name)) end write(joinpath(dest, "index.html"), record_page(name, projects, rec)) end diff --git a/src/deposit.jl b/src/deposit.jl index ed6ecf2..b02fba8 100644 --- a/src/deposit.jl +++ b/src/deposit.jl @@ -145,7 +145,12 @@ function readme(e) return String(take!(io)) end +# A face is a directory; `Pinax.render` and `Pinax.report` return the file they wrote in it +# (`index.html`, `agent.json`), so a file stands for its directory. +face_dir(path) = isfile(path) ? dirname(path) : path + function copy_tree(src, dest) + isdir(src) || error("$src is not a directory") for (dir, _, files) in walkdir(src), f in files f in SKIP && continue target = joinpath(dest, relpath(joinpath(dir, f), src)) @@ -269,23 +274,23 @@ function deposit( incoming = joinpath(reg, "_incoming", rev) ispath(incoming) && error("$incoming exists") mkpath(incoming) - copy_tree(gallery, joinpath(incoming, "gallery")) - copy_tree(agent, joinpath(incoming, "agent")) - for (dest, src) in repro - mkpath(dirname(joinpath(incoming, "repro", dest))) - cp(src, joinpath(incoming, "repro", dest)) - end - open(io -> TOML.print(io, entry; sorted=true), joinpath(incoming, "entry.toml"), "w") - write(joinpath(incoming, "README.md"), readme(entry)) - if provenance !== nothing - try - write_provenance!(incoming; provenance...) - catch - rm(incoming; recursive=true, force=true) # nothing half-written is left behind - rethrow() + try + copy_tree(face_dir(gallery), joinpath(incoming, "gallery")) + copy_tree(face_dir(agent), joinpath(incoming, "agent")) + for (dest, src) in repro + mkpath(dirname(joinpath(incoming, "repro", dest))) + cp(src, joinpath(incoming, "repro", dest)) end + open( + io -> TOML.print(io, entry; sorted=true), joinpath(incoming, "entry.toml"), "w" + ) + write(joinpath(incoming, "README.md"), readme(entry)) + provenance === nothing || write_provenance!(incoming; provenance...) + write_sums(incoming) # last: its presence means "complete" + catch + rm(incoming; recursive=true, force=true) # nothing half-written is left behind + rethrow() end - write_sums(incoming) # last: its presence means "complete" final = joinpath(revroot, rev) mkpath(revroot) diff --git a/src/provenance.jl b/src/provenance.jl index 3cc7368..bfc7fd0 100644 --- a/src/provenance.jl +++ b/src/provenance.jl @@ -239,6 +239,16 @@ function check_provenance(r, revdir) obs = load(r, obs_path) obs === nothing && continue binding = get(obs, "binding", nothing) + # Observation version 1 looked for code defined in Main where Julia records none made at + # run time, so its match can hide a script's code (DataVault 0.8.6 fixed the check). + get(obs, "observation_version", nothing) == 1 && + binding == "loaded-matches-disk" && + warn!( + r, + obs_path, + "observation version 1 could not see code defined in Main; its " * + "`loaded-matches-disk` does not rule out a script's code", + ) binding_of_token[t] = string(binding) binding in BINDINGS || err!( r, diff --git a/test/test_deposit.jl b/test/test_deposit.jl index a732fc6..1908eb8 100644 --- a/test/test_deposit.jl +++ b/test/test_deposit.jl @@ -1,5 +1,36 @@ # deposit: revisions go in through a binding, and a revision that does not validate never lands. +@testset "deposit: the file a render returns stands for its directory" begin + with_git_fixture() do root, binding, src + res = deposit( + binding; + gallery=joinpath(src.gallery, "index.html"), + agent=joinpath(src.agent, "agent.json"), + doc=DOC, + source_repo=root, + push=false, + ) + @test isfile(joinpath(res.dir, "gallery", "index.html")) + @test isfile(joinpath(res.dir, "agent", "agent.json")) + @test isempty(first(Archeion.validate(root)).errors) + end + with_git_fixture() do root, binding, src + e = attempt( + () -> deposit( + binding; + gallery=joinpath(root, "no-such-dir"), + agent=src.agent, + doc=DOC, + source_repo=root, + push=false, + ), + ) + @test e isa ErrorException && occursin("is not a directory", e.msg) + incoming = joinpath(root, "_incoming") + @test commits(root) == 1 && (!isdir(incoming) || isempty(readdir(incoming))) + end +end + @testset "deposit" begin with_git_fixture() do root, binding, src res = deposit(binding; src..., doc=DOC, source_repo=root, push=false) diff --git a/test/test_provenance.jl b/test/test_provenance.jl index 62c496d..3b720f9 100644 --- a/test/test_provenance.jl +++ b/test/test_provenance.jl @@ -7,7 +7,7 @@ using DataVault # contents kept), and one observation of it with the given binding. const OBS = "obs1-20260922T000000Z-1a2b-0123456789abcdef" -function synthetic_store(; binding="loaded-matches-disk") +function synthetic_store(; binding="loaded-matches-disk", version=2) dir = mktempdir() blob = "module P\nend\n" bsha = bytes2hex(sha256(blob)) @@ -24,7 +24,8 @@ function synthetic_store(; binding="loaded-matches-disk") mkpath(observations) write( joinpath(observations, "$token.toml"), - "token = \"$token\"\nsource = \"$id\"\nbinding = \"$binding\"\nbinding_reasons = []\n", + "observation_version = $version\ntoken = \"$token\"\nsource = \"$id\"\n" * + "binding = \"$binding\"\nbinding_reasons = []\n", ) return (; dir, observations, sources, token, id, bsha) end @@ -108,6 +109,10 @@ rewrite!(path, f) = write(path, f(read(path, String))) Archeion.build(root, site) rec = relpath(dirname(dirname(res.dir)), root) page = read(joinpath(site, rec, "index.html"), String) + served = joinpath(site, relpath(res.dir, root)) + @test isfile(joinpath(served, "provenance.toml")) && + isdir(joinpath(served, "gallery")) + @test !ispath(joinpath(served, "provenance")) && !ispath(joinpath(served, "repro")) @test occursin("3 points: 2 read as recorded", page) && occursin("1 unrecorded", page) @test occursin("code loaded-matches-disk 2, unknown 1", page) @@ -151,6 +156,21 @@ end end end +@testset "provenance: a version-1 match is not taken at its word" begin + store = synthetic_store(; version=1) + deposited([point("k1", store.token)]; store) do root, res + r, _ = Archeion.validate(root) + @test isempty(r.errors) + @test mentions( + r.warnings, "observation version 1 could not see code defined in Main" + ) + end + store = synthetic_store(; version=1, binding="unverified") + deposited([point("k1", store.token)]; store) do root, res + @test isempty(first(Archeion.validate(root)).warnings) + end +end + @testset "provenance: a name that is not a token never becomes a path" begin deposited([point("k1", "../../escape")]) do root, res @test res isa ErrorException && occursin("not an observation token", res.msg) From 7f90f1fe8f583482c5dc5b733e2410d4df839cfd Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Tue, 22 Sep 2026 11:44:16 +0000 Subject: [PATCH 2/4] Count loaded-matches-disk as unverified; keep the original error on cleanup failure A match cannot be shown from inside the computing process, and DataVault 0.8.6 stops writing one (DataVault#57). An earlier observation's loaded-matches-disk is now warned on and counted as unverified, in the deposited summary and in the check, rather than only warned on while the count kept it. A deposit whose cleanup rm fails rethrows the step's own error, not the rm's. Co-Authored-By: Claude Opus 5 (1M context) --- SPEC.md | 11 ++++++----- src/deposit.jl | 10 +++++++--- src/provenance.jl | 25 +++++++++++++------------ test/test_provenance.jl | 27 ++++++++++++++++----------- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/SPEC.md b/SPEC.md index 12b0e4f..f7b2cf6 100644 --- a/SPEC.md +++ b/SPEC.md @@ -180,15 +180,16 @@ key file read_sha256 result_sha256 observation completed_at when kept, at `repro/blobs/`. Paths use 32 hex (128 bits) to stay within R3; the files keep the full digests, and a checker compares full digests. - An observation's `binding` is how far that process's loaded code was checked against its - snapshot: `loaded-matches-disk` (every loaded source of the checked roots matched it), - `loaded-differs-from-disk`, or `unverified` (with `binding_reasons`). An observation is a disk - state; only its binding speaks about the code that ran, and `unverified` claims nothing. + snapshot: `loaded-differs-from-disk` (a loaded package's sources are not what the snapshot + holds) or `unverified` (with `binding_reasons`). An observation is a disk state, and + `unverified` claims nothing about the code that ran. A `loaded-matches-disk`, which earlier + data stores wrote, is read and counted as `unverified`: a match cannot be shown from inside the + computing process, since code defined outside a package leaves no trace to check it against. - A depositor refuses a row whose `read_sha256` differs from its `result_sha256` unless told to let it in, which is then recorded as `allow_mismatch = true`. A validator requires the summary to match the table, every named token to be held or listed as missing, every binding to be one of the three values, and every snapshot to hash to its id; it warns on rows that read other bytes, on - missing observations, and on a `loaded-matches-disk` from an observation of - `observation_version = 1`, whose check for code defined in a script never fired. + missing observations, and on every `loaded-matches-disk`. - When `provenance.toml` exists, `repro/observations/`, `repro/sources/` and `repro/blobs/` are its own. diff --git a/src/deposit.jl b/src/deposit.jl index b02fba8..ca09421 100644 --- a/src/deposit.jl +++ b/src/deposit.jl @@ -287,9 +287,13 @@ function deposit( write(joinpath(incoming, "README.md"), readme(entry)) provenance === nothing || write_provenance!(incoming; provenance...) write_sums(incoming) # last: its presence means "complete" - catch - rm(incoming; recursive=true, force=true) # nothing half-written is left behind - rethrow() + catch e + try + rm(incoming; recursive=true, force=true) # nothing half-written is left behind + catch cleanup + @warn "could not remove $incoming" exception = cleanup + end + rethrow(e) end final = joinpath(revroot, rev) diff --git a/src/provenance.jl b/src/provenance.jl index bfc7fd0..29441d9 100644 --- a/src/provenance.jl +++ b/src/provenance.jl @@ -10,6 +10,11 @@ const POINT_COLUMNS = ( "key", "file", "read_sha256", "result_sha256", "observation", "completed_at" ) const BINDINGS = Set(["loaded-matches-disk", "loaded-differs-from-disk", "unverified"]) + +# A match cannot be shown from inside the computing process (code defined in a script, a closure, +# or a method added to Base leaves no trace to check), and DataVault 0.8.6 no longer writes one. An +# earlier observation's `loaded-matches-disk` is therefore counted as what it can support. +_counted(binding) = binding == "loaded-matches-disk" ? "unverified" : binding const POINTS_FILE = "provenance/points.tsv" # Content-addressed names are cut to 32 hex (128 bits) in paths, so they stay within R3 (64 per @@ -112,7 +117,7 @@ function write_provenance!( push!(present, t) _copy_if_absent(src, joinpath(revdir, "repro", "observations", "$t.toml")) obs = TOML.parsefile(src) - binding_of_token[t] = get(obs, "binding", "unknown") + binding_of_token[t] = _counted(get(obs, "binding", "unknown")) if source_contents # the Project/Manifest the process ran with, when it stored them for sha in values(get(obs, "environment", Dict())) blob = joinpath(sources_dir, "blobs", string(sha)) @@ -239,17 +244,13 @@ function check_provenance(r, revdir) obs = load(r, obs_path) obs === nothing && continue binding = get(obs, "binding", nothing) - # Observation version 1 looked for code defined in Main where Julia records none made at - # run time, so its match can hide a script's code (DataVault 0.8.6 fixed the check). - get(obs, "observation_version", nothing) == 1 && - binding == "loaded-matches-disk" && - warn!( - r, - obs_path, - "observation version 1 could not see code defined in Main; its " * - "`loaded-matches-disk` does not rule out a script's code", - ) - binding_of_token[t] = string(binding) + binding == "loaded-matches-disk" && warn!( + r, + obs_path, + "`loaded-matches-disk` cannot rule out code defined outside a package (a script, a " * + "closure, a method added to Base); it is counted as `unverified`", + ) + binding_of_token[t] = _counted(string(binding)) binding in BINDINGS || err!( r, obs_path, diff --git a/test/test_provenance.jl b/test/test_provenance.jl index 3b720f9..fb8b951 100644 --- a/test/test_provenance.jl +++ b/test/test_provenance.jl @@ -7,7 +7,7 @@ using DataVault # contents kept), and one observation of it with the given binding. const OBS = "obs1-20260922T000000Z-1a2b-0123456789abcdef" -function synthetic_store(; binding="loaded-matches-disk", version=2) +function synthetic_store(; binding="unverified", version=1) dir = mktempdir() blob = "module P\nend\n" bsha = bytes2hex(sha256(blob)) @@ -93,7 +93,7 @@ rewrite!(path, f) = write(path, f(read(path, String))) "read_differs_from_result" => 0, "result_unknown" => 1, ) - @test p["bindings"] == Dict("loaded-matches-disk" => 2, "unknown" => 1) + @test p["bindings"] == Dict("unverified" => 2, "unknown" => 1) @test p["observations"] == [t] && isempty(p["missing_observations"]) rows = readlines(joinpath(res.dir, "provenance", "points.tsv")) @test first.(split.(rows[2:end], '\t')) == ["k1", "k2", "k3"] @@ -115,7 +115,7 @@ rewrite!(path, f) = write(path, f(read(path, String))) @test !ispath(joinpath(served, "provenance")) && !ispath(joinpath(served, "repro")) @test occursin("3 points: 2 read as recorded", page) && occursin("1 unrecorded", page) - @test occursin("code loaded-matches-disk 2, unknown 1", page) + @test occursin("code unknown 1, unverified 2", page) rm(dirname(site); recursive=true) end end @@ -156,18 +156,21 @@ end end end -@testset "provenance: a version-1 match is not taken at its word" begin - store = synthetic_store(; version=1) +@testset "provenance: a match is not taken at its word" begin + store = synthetic_store(; binding="loaded-matches-disk") deposited([point("k1", store.token)]; store) do root, res r, _ = Archeion.validate(root) @test isempty(r.errors) - @test mentions( - r.warnings, "observation version 1 could not see code defined in Main" - ) + @test mentions(r.warnings, "it is counted as `unverified`") + p = TOML.parsefile(joinpath(res.dir, "provenance.toml")) + @test p["bindings"] == Dict("unverified" => 1) # never counted as a match end - store = synthetic_store(; version=1, binding="unverified") + store = synthetic_store(; binding="loaded-differs-from-disk") deposited([point("k1", store.token)]; store) do root, res - @test isempty(first(Archeion.validate(root)).warnings) + r, _ = Archeion.validate(root) + @test isempty(r.warnings) + @test TOML.parsefile(joinpath(res.dir, "provenance.toml"))["bindings"] == + Dict("loaded-differs-from-disk" => 1) end end @@ -202,7 +205,9 @@ end v = broken() do d obs = joinpath(d, "repro", "observations", "$t.toml") - rewrite!(obs, s -> replace(s, "loaded-matches-disk" => "trust-me")) + rewrite!( + obs, s -> replace(s, "binding = \"unverified\"" => "binding = \"trust-me\"") + ) end @test mentions(v.errors, "\"trust-me\" is not one of") @test mentions(v.errors, "`bindings` does not match") From 3ca7d298f0e1b83e9ebdd692357f3db324dfca92 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 23 Sep 2026 03:00:37 +0000 Subject: [PATCH 3/4] Give the cleanup its own function, and test that its failure is only a warning Co-Authored-By: Claude Opus 5 (1M context) --- src/deposit.jl | 17 ++++++++++++----- test/test_deposit.jl | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/deposit.jl b/src/deposit.jl index ca09421..034db61 100644 --- a/src/deposit.jl +++ b/src/deposit.jl @@ -149,6 +149,17 @@ end # (`index.html`, `agent.json`), so a file stands for its directory. face_dir(path) = isfile(path) ? dirname(path) : path +# Remove a revision being built. A failure here must not replace the failure that got us here: +# the caller is about to rethrow what actually went wrong. +function discard!(dir) + try + rm(dir; recursive=true, force=true) + catch e + @warn "could not remove $dir" exception = e + end + return nothing +end + function copy_tree(src, dest) isdir(src) || error("$src is not a directory") for (dir, _, files) in walkdir(src), f in files @@ -288,11 +299,7 @@ function deposit( provenance === nothing || write_provenance!(incoming; provenance...) write_sums(incoming) # last: its presence means "complete" catch e - try - rm(incoming; recursive=true, force=true) # nothing half-written is left behind - catch cleanup - @warn "could not remove $incoming" exception = cleanup - end + discard!(incoming) # nothing half-written is left behind rethrow(e) end diff --git a/test/test_deposit.jl b/test/test_deposit.jl index 1908eb8..e0eac5d 100644 --- a/test/test_deposit.jl +++ b/test/test_deposit.jl @@ -1,5 +1,20 @@ # deposit: revisions go in through a binding, and a revision that does not validate never lands. +@testset "deposit: a cleanup that fails keeps the failure it was cleaning up after" begin + parent = mktempdir() + dir = joinpath(parent, "rev") + mkpath(dir) + write(joinpath(dir, "entry.toml"), "x") + chmod(parent, 0o500) # the entry cannot be unlinked + try + @test_logs (:warn, r"could not remove") Archeion.discard!(dir) + @test isdir(dir) # and the caller still rethrows its own + finally + chmod(parent, 0o700) + rm(parent; recursive=true, force=true) + end +end + @testset "deposit: the file a render returns stands for its directory" begin with_git_fixture() do root, binding, src res = deposit( From 634c94181aa1293bfbf8fdbb136f25f8df208e43 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 23 Sep 2026 03:04:14 +0000 Subject: [PATCH 4/4] Test that deposit puts `repro` files under repro/ Co-Authored-By: Claude Opus 5 (1M context) --- test/test_deposit.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/test_deposit.jl b/test/test_deposit.jl index e0eac5d..03db734 100644 --- a/test/test_deposit.jl +++ b/test/test_deposit.jl @@ -1,5 +1,23 @@ # deposit: revisions go in through a binding, and a revision that does not validate never lands. +@testset "deposit: `repro` puts named files under repro/" begin + with_git_fixture() do root, binding, src + script = joinpath(mktempdir(), "run.jl") + write(script, "# the script that made it\n") + res = deposit( + binding; + src..., + doc=DOC, + source_repo=root, + push=false, + repro=Dict("scripts/run.jl" => script), + ) + @test read(joinpath(res.dir, "repro", "scripts", "run.jl"), String) == + "# the script that made it\n" + @test isempty(first(Archeion.validate(root)).errors) + end +end + @testset "deposit: a cleanup that fails keeps the failure it was cleaning up after" begin parent = mktempdir() dir = joinpath(parent, "rev")