From 3f1da3954ef94299d5d09aa9cf63c8f823e9f48b Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Fri, 25 Sep 2026 04:32:53 +0000 Subject: [PATCH 1/4] Keep what a recomputation needs, and say which Julia and BLAS ran (0.8.7) A revision was shown on 2026-09-25 not to be rebuildable from itself: with an empty depot, an empty HOME and no network, a restore needed git-URL packages that only an SSH remote could give back, and `julia` started another version than the one recorded. The same day, one point recomputed at BLAS thread counts 1, 8 and 18 gave three different last bits, and the count was nowhere in the observation. - A run-start observation keeps every package the process loaded from a depot as a `depot` root, whose head is the tree the Manifest pins. A render does not by default: its plotting stack is not what a recomputation needs. - Contents are kept for any file up to `materialize_limit` (1 MiB), not only .jl and .toml, which still are at any size. An extension list dropped the tables and scripts a package reads, without a word. - The study is the active environment's directory when the config lies in it, so a repository of many studies captures one; otherwise it is the config's repository, as before. - The vault's own output is never source, even inside the study. - `julia` records the binary's digest, bindir, platform, CPU, BLAS's thread count and its libraries. Co-Authored-By: Claude Opus 5.5 (1M context) --- Project.toml | 4 +- src/DataVault.jl | 1 + src/provenance/observe.jl | 167 +++++++++++++++++++++++++++++-------- test/vault/test_observe.jl | 112 +++++++++++++++++++++++-- 4 files changed, 244 insertions(+), 40 deletions(-) diff --git a/Project.toml b/Project.toml index 526c7d2..19c8bdd 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DataVault" uuid = "23f5f8f6-b4da-40ee-8c72-c53b6c5de94f" -version = "0.8.6" +version = "0.8.7" authors = ["sota shimozono "] [deps] @@ -8,6 +8,7 @@ CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ParamIO = "938a3ac2-d340-473c-bcf1-88af577e4ccf" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -20,6 +21,7 @@ CRC32c = "1.11" Dates = "1.11.0" JLD2 = "0.6.4" JSON3 = "1" +LinearAlgebra = "1.11" Logging = "1.11" # 0.4.11 is the floor, not a preference: `artifact_identity` / `artifact_key` (what `artifact!` # stores by) arrived there; `float_format` and the 3-arg `format_path` arrived in 0.4.0. diff --git a/src/DataVault.jl b/src/DataVault.jl index 29111fa..d8fa7c3 100644 --- a/src/DataVault.jl +++ b/src/DataVault.jl @@ -28,6 +28,7 @@ using JLD2, TOML, Dates, Printf using SHA using CRC32c: crc32c using Random: Random +using LinearAlgebra: LinearAlgebra using JSON3 using ParamIO diff --git a/src/provenance/observe.jl b/src/provenance/observe.jl index 98fb0a0..abfd845 100644 --- a/src/provenance/observe.jl +++ b/src/provenance/observe.jl @@ -16,7 +16,8 @@ const SOURCE_RECIPE = "src1" const OBSERVATION_VERSION = 1 -const MATERIALIZE_EXTENSIONS = (".jl", ".toml") # contents kept; every other file inventoried only +const MATERIALIZE_EXTENSIONS = (".jl", ".toml") # contents always kept, at any size up to the hash limit +const DEFAULT_MATERIALIZE_LIMIT = 1024^2 # any other file: contents kept up to this size const DEFAULT_HASH_LIMIT = 64 * 1024^2 # larger files are inventoried without a digest const ENV_RECORDED = ( "JULIA_NUM_THREADS", @@ -55,30 +56,70 @@ _inside(path, dir) = startswith(_real(path), rstrip(_real(dir), '/') * "/") # ── roots ───────────────────────────────────────────────────────────────────────────────────── -# The config's repository (or directory), and every `path` dependency of the active environment -# that is not already inside it. Names are logical — no absolute path enters the snapshot. -function _source_roots(vault::Vault) +# The study, every `path` dependency of the active environment that is not already inside it, and — +# when `depot` — every package this process loaded from a depot. Names are logical: no absolute +# path enters the snapshot. +# +# The study is the active environment's directory when the config lies inside it (a study that +# carries its own Project.toml, so that a repository of many studies does not capture them all), +# and otherwise the config's repository, or its directory outside git. +function _source_roots(vault::Vault; depot::Bool=false) cfg = dirname(abspath(vault.config_path)) top = _git_read(cfg, "rev-parse", "--show-toplevel") - roots = [ - if top === nothing - (name="config", dir=cfg, kind=:plain) - else - (name="config", dir=top, kind=:git) - end, - ] + project = Base.active_project() + envdir = project === nothing ? nothing : dirname(abspath(project)) + study = if envdir !== nothing && (_inside(cfg, envdir) || _real(cfg) == _real(envdir)) + envdir + else + something(top, cfg) + end + roots = Any[(name="config", dir=study, kind=top === nothing ? :plain : :git, tree="")] for dep in _path_dependencies() any(r -> _inside(dep.path, r.dir) || _real(dep.path) == _real(r.dir), roots) && continue ingit = _git_read(dep.path, "rev-parse", "--show-toplevel") !== nothing push!( roots, - (name="pkg:$(dep.name):$(dep.uuid)", dir=dep.path, kind=ingit ? :git : :plain), + ( + name="pkg:$(dep.name):$(dep.uuid)", + dir=dep.path, + kind=ingit ? :git : :plain, + tree="", + ), ) end + depot && append!(roots, _depot_roots(roots)) return roots end +# Every package this process loaded from a depot (`packages//`), with the tree hash the +# active Manifest pins it to. These are what a registered or git-URL dependency is: a tree that +# only a registry, a package server or a git remote can give back, so the snapshot keeps it. +function _depot_roots(existing) + manifest = _active_manifest() + manifest === nothing && return Any[] + pinned = Dict{String,String}() + for (_, entries) in get(TOML.parsefile(manifest), "deps", Dict{String,Any}()), + e in entries + + haskey(e, "git-tree-sha1") && + haskey(e, "uuid") && + (pinned[e["uuid"]] = e["git-tree-sha1"]) + end + depots = [joinpath(_real(d), "packages") for d in DEPOT_PATH if isdir(d)] + out = Any[] + for (id, origin) in Base.pkgorigins + (id.uuid === nothing || origin.path === nothing) && continue + tree = get(pinned, string(id.uuid), nothing) + tree === nothing && continue + any(d -> _inside(origin.path, d), depots) || continue + dir = dirname(dirname(origin.path)) # /src/.jl → + any(r -> _real(r.dir) == _real(dir) || _inside(dir, r.dir), existing) && continue + push!(out, (name="pkg:$(id.name):$(id.uuid)", dir=dir, kind=:depot, tree=tree)) + end + return sort!(out; by=r -> r.name) +end + const _PathDep = NamedTuple{(:name, :uuid, :path),NTuple{3,String}} function _active_manifest()::Union{String,Nothing} @@ -108,20 +149,28 @@ end # ── inventory ───────────────────────────────────────────────────────────────────────────────── -function _root_files(root)::Union{Vector{String},Nothing} +# Relative to the root. Anything under `exclude` (the vault's own output) is not source, even when +# it sits inside the study and nothing ignores it. +function _root_files(root; exclude=nothing)::Union{Vector{String},Nothing} + keep(rel) = exclude === nothing || !_inside(joinpath(root.dir, rel), exclude) if root.kind === :git out = _git_read( root.dir, "ls-files", "-z", "--cached", "--others", "--exclude-standard" ) out === nothing && return nothing - return sort!(unique!(filter!(!isempty, String.(split(out, '\0'))))) + return sort!(filter!(keep, unique!(filter!(!isempty, String.(split(out, '\0')))))) end files = String[] for (dir, dirs, fs) in walkdir(root.dir) filter!(d -> d != ".git", dirs) + # Do not walk into the output at all: it can be far larger than the study. + exclude === nothing || filter!(dirs) do d + p = joinpath(dir, d) + return !(_real(p) == _real(exclude) || _inside(p, exclude)) + end append!(files, relpath(joinpath(dir, f), root.dir) for f in fs) end - return sort!(files) + return sort!(filter!(keep, files)) end struct SourceEntry @@ -135,7 +184,15 @@ struct SourceEntry full::String end -function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry +# A file's contents are kept when it is `.jl`/`.toml` (at any size up to the hash limit) or no +# larger than `materialize_limit`: the snapshot must hold what a package reads besides its code (a +# table, a template, a script), and an extension list drops those without a word. +function _materialize(rel, size, limit) + return size <= limit || + any(ext -> endswith(lowercase(rel), ext), MATERIALIZE_EXTENSIONS) +end + +function _entry(root, rel, hash_limit, blobs, notes; materialize_limit)::SourceEntry full = joinpath(root.dir, rel) st = lstat(full) if islink(st) @@ -160,8 +217,7 @@ function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry end bytes = read(full) sha = bytes2hex(sha256(bytes)) - any(ext -> endswith(lowercase(rel), ext), MATERIALIZE_EXTENSIONS) && - (blobs[sha] = bytes) + _materialize(rel, length(bytes), materialize_limit) && (blobs[sha] = bytes) return SourceEntry( root.name, rel, "file", mode, length(bytes), sha, crc32c(bytes), full ) @@ -172,17 +228,22 @@ function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry return SourceEntry(root.name, rel, "missing", "-", 0, "", 0x00000000, full) end -function _inventory(roots; hash_limit::Integer) +function _inventory( + roots; hash_limit::Integer, materialize_limit::Integer=0, exclude=nothing +) entries = SourceEntry[] blobs = Dict{String,Vector{UInt8}}() notes = String[] for root in roots - files = _root_files(root) + files = _root_files(root; exclude) if files === nothing push!(notes, "$(root.name): files could not be listed") continue end - append!(entries, _entry(root, rel, hash_limit, blobs, notes) for rel in files) + append!( + entries, + _entry(root, rel, hash_limit, blobs, notes; materialize_limit) for rel in files + ) end return entries, blobs, notes end @@ -352,8 +413,16 @@ function _root_record(root, loaded::String)::Dict{String,Any} observed = _git_observe(root.dir) record["head"] = observed.commit record["object_format"] = observed.object_format - porcelain = _git_read(root.dir, "status", "--porcelain", "--untracked-files=all") + porcelain = _git_read( + root.dir, "status", "--porcelain", "--untracked-files=all", "--", "." + ) record["dirty"] = porcelain === nothing ? "unknown" : string(!isempty(porcelain)) + elseif root.kind === :depot + # The tree the Manifest pins. Whether the directory still hashes to it is for a restore + # to check: a depot is not written to after install, but nothing here proves it. + record["head"] = root.tree + record["object_format"] = "sha1" + record["dirty"] = "unknown" else record["head"] = record["object_format"] = record["dirty"] = "unknown" end @@ -376,7 +445,8 @@ end function _julia_record()::Dict{String,Any} opts = Base.JLOptions() - return Dict{String,Any}( + exe = joinpath(Sys.BINDIR, Base.julia_exename()) + record = Dict{String,Any}( "version" => string(VERSION), "commit" => Base.GIT_VERSION_INFO.commit, "image_file" => opts.image_file == C_NULL ? "" : unsafe_string(opts.image_file), @@ -384,40 +454,71 @@ function _julia_record()::Dict{String,Any} "opt_level" => Int(opts.opt_level), "fast_math" => Int(opts.fast_math), "threads" => Threads.nthreads(), + # Which binary, not only which version: a launcher (juliaup) given the same command in + # another HOME starts another Julia. The digest is of the binary itself. + "bindir" => Sys.BINDIR, + "platform" => Base.BinaryPlatforms.triplet(Base.BinaryPlatforms.HostPlatform()), + "cpu_name" => Sys.CPU_NAME, + # BLAS's own thread count, whatever set it: results differ in the last bits between + # counts, so a bitwise comparison is only meaningful at the one recorded here. + "blas_threads" => LinearAlgebra.BLAS.get_num_threads(), + "blas_libraries" => + [basename(l.libname) for l in LinearAlgebra.BLAS.get_config().loaded_libs], ) + isfile(exe) && (record["executable_sha256"] = bytes2hex(open(sha256, exe))) + return record end """ - observe_sources(vault; phase = "run-start", process = Dict(), hash_limit = 64 MiB) -> token + observe_sources(vault; phase = "run-start", process = Dict(), hash_limit = 64 MiB, + materialize_limit = 1 MiB, depot_packages = (phase == "run-start")) -> token + +Observe the source roots this process can see, store the snapshot (once per distinct content) and +an observation record, and return the record's token for [`mark_done!`](@ref)'s `observation`. -Observe the source roots this process can see — the config's repository and every `path` -dependency of the active environment — store the snapshot (once per distinct content) and an -observation record, and return the record's token for [`mark_done!`](@ref)'s `observation`. +The roots are the study (the active environment's directory when the config lies inside it, +otherwise the config's repository), every `path` dependency of the active environment, and, when +`depot_packages`, every package this process loaded from a depot, recorded as a `depot` root whose +`head` is the tree hash the Manifest pins. The vault's own output directory is never source. The record says when (`observed_at`, `phase`), where (host, pid, and whatever `process` adds, such as a worker id), which snapshot (`source`), each root's git HEAD and whether it was dirty, the Julia -build and a fixed list of environment variables, and the **binding**: how far the code this process -has loaded was checked against the snapshot (see [`binding_of`](@ref)). File contents are stored -only for `.jl` and `.toml` files; every other file is inventoried by size and digest. +build (with the binary's digest, the platform and BLAS's thread count) and a fixed list of +environment variables, and the **binding**: how far the code this process has loaded was checked +against the snapshot (see [`binding_of`](@ref)). File contents are stored for `.jl` and `.toml` +files and for any file up to `materialize_limit`; every file is inventoried by size and digest. + +Depot packages are kept at `run-start` only by default: that is the process that computed, and a +render's plotting stack is large and not what a recomputation needs. """ function observe_sources( vault::Vault; phase::AbstractString="run-start", process::AbstractDict=Dict{String,Any}(), hash_limit::Integer=DEFAULT_HASH_LIMIT, + materialize_limit::Integer=DEFAULT_MATERIALIZE_LIMIT, + depot_packages::Bool=(phase == "run-start"), )::String _refuse_if_readonly(vault, "observe_sources") observed_at = _utc_stamp() - roots = _source_roots(vault) - entries, blobs, notes = _inventory(roots; hash_limit) + roots = _source_roots(vault; depot=depot_packages) + entries, blobs, notes = _inventory( + roots; hash_limit, materialize_limit, exclude=vault.outdir + ) complete = !any(e -> e.sha256 == "skipped" || e.type == "dir", entries) && !any(n -> occursin("could not be listed", n), notes) state = Dict{String,Any}( "recipe" => SOURCE_RECIPE, - "roots" => [Dict("name" => r.name, "kind" => String(r.kind)) for r in roots], + "roots" => [ + merge( + Dict("name" => r.name, "kind" => String(r.kind)), + isempty(r.tree) ? Dict{String,String}() : Dict("tree" => r.tree), + ) for r in roots + ], "inventory_complete" => complete, "materialized" => collect(MATERIALIZE_EXTENSIONS), + "materialize_limit" => materialize_limit, "notes" => notes, ) source = _publish_snapshot(vault, _files_tsv(entries), state, blobs) diff --git a/test/vault/test_observe.jl b/test/vault/test_observe.jl index 473beda..b999444 100644 --- a/test/vault/test_observe.jl +++ b/test/vault/test_observe.jl @@ -67,13 +67,111 @@ config_root(rec) = only(r for r in rec["roots"] if r["name"] == "config") end end -@testset "observe_sources: only .jl and .toml contents are kept" begin +@testset "observe_sources: contents are kept by size, and .jl/.toml at any size" begin + sha(file) = bytes2hex(sha256(read(file))) with_observed_repo() do t observe_sources(t.vault) blobs = joinpath(obs_dir(t), "sources", "blobs") - sha(file) = bytes2hex(sha256(read(file))) + @test isfile(joinpath(blobs, sha(joinpath(t.pkg, "src", "$(t.name).jl")))) + # A data file a package reads is source too; an extension list would drop it silently. + @test isfile(joinpath(blobs, sha(joinpath(t.repo, "notes.dat")))) + end + with_observed_repo() do t + r = record(t, observe_sources(t.vault; materialize_limit=4)) + blobs = joinpath(obs_dir(t), "sources", "blobs") @test isfile(joinpath(blobs, sha(joinpath(t.pkg, "src", "$(t.name).jl")))) @test !isfile(joinpath(blobs, sha(joinpath(t.repo, "notes.dat")))) + state = TOML.parsefile(joinpath(obs_dir(t), "sources", r["source"], "state.toml")) + @test state["materialize_limit"] == 4 + end +end + +@testset "observe_sources: the vault's own output inside the study is not source" begin + repo = mktempdir() + try + cp(_OBS_CFG, joinpath(repo, "study.toml")) + out = joinpath(repo, "out") # inside the study, ignored by nothing + v = Vault(joinpath(repo, "study.toml"); outdir=out) + k = DataVault.keys(v)[1] + mark_done!(v, k; result=DataVault.save!(v, k, Dict("x" => 1.0))) + store = joinpath(out, ".datavault", "test_study") + r = TOML.parsefile(joinpath(store, "observations", "$(observe_sources(v)).toml")) + tsv = read(joinpath(store, "sources", r["source"], "files.tsv"), String) + @test occursin("config\tstudy.toml\t", tsv) + @test !occursin("config\tout/", tsv) + finally + rm(repo; recursive=true, force=true) + end +end + +@testset "observe_sources: a study with its own environment is its own root" begin + with_observed_repo() do t + study = joinpath(t.repo, "studies", "one") + mkpath(study) + cp(_OBS_CFG, joinpath(study, "study.toml")) + write(joinpath(study, "Project.toml"), "[deps]\n") + write(joinpath(t.repo, "studies", "other.jl"), "x = 1\n") # a neighbour, not this study + git!(t.repo, "add", "-A") + git!(t.repo, "commit", "-qm", "studies") + before = Base.active_project() + Base.set_active_project(joinpath(study, "Project.toml")) + try + v = Vault(joinpath(study, "study.toml"); outdir=t.out) + r = record(t, observe_sources(v)) + tsv = read(joinpath(obs_dir(t), "sources", r["source"], "files.tsv"), String) + @test occursin("config\tstudy.toml\t", tsv) + @test occursin("config\tProject.toml\t", tsv) + @test !occursin("other.jl", tsv) && !occursin("notes.dat", tsv) + c = config_root(r) + @test c["kind"] == "git" + @test c["head"] == readchomp(`git -C $(t.repo) rev-parse HEAD`) + @test c["dirty"] == "false" + write(joinpath(t.repo, "notes.dat"), "changed outside the study") + @test config_root(record(t, observe_sources(v)))["dirty"] == "false" + finally + Base.set_active_project(before) + end + end +end + +@testset "observe_sources: the packages a computing process loaded are kept" begin + with_observed_repo() do t + manifest = DataVault._active_manifest() + pinned = Dict{String,String}() + if manifest !== nothing + for (_, es) in TOML.parsefile(manifest)["deps"], e in es + haskey(e, "git-tree-sha1") && (pinned[e["uuid"]] = e["git-tree-sha1"]) + end + end + # JLD2 is loaded by DataVault itself, from a depot, whenever these tests run. + uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" + r = record(t, observe_sources(t.vault)) + if haskey(pinned, uuid) + j = only(x for x in r["roots"] if x["name"] == "pkg:JLD2:$uuid") + @test j["kind"] == "depot" && j["head"] == pinned[uuid] + snap = joinpath(obs_dir(t), "sources", r["source"]) + @test occursin( + "pkg:JLD2:$uuid\tsrc/JLD2.jl\tfile", + read(joinpath(snap, "files.tsv"), String), + ) + state = TOML.parsefile(joinpath(snap, "state.toml")) + @test any(x -> get(x, "tree", "") == j["head"], state["roots"]) + else + @test_broken haskey(pinned, uuid) # JLD2 is not pinned by a tree here + end + # A render is not the computing process: its plotting stack is not kept by default. + rr = record(t, observe_sources(t.vault; phase="render")) + @test !any(x -> x["kind"] == "depot", rr["roots"]) + end +end + +@testset "observe_sources: which Julia binary, and BLAS's thread count" begin + with_observed_repo() do t + j = record(t, observe_sources(t.vault))["julia"] + @test j["blas_threads"] == DataVault.LinearAlgebra.BLAS.get_num_threads() + @test occursin(r"^[0-9a-f]{64}$", j["executable_sha256"]) + @test j["bindir"] == Sys.BINDIR + @test !isempty(j["platform"]) && !isempty(j["blas_libraries"]) end end @@ -147,10 +245,12 @@ end k = DataVault.keys(v)[1] token = observe_sources(v) mark_done!(v, k; result=DataVault.save!(v, k, Dict("x" => 1.0)), observation=token) - line(key) = only( - l for - l in eachline(DataVault._done_file(v, key)) if startswith(l, "observation=") - ) + function line(key) + return only( + l for + l in eachline(DataVault._done_file(v, key)) if startswith(l, "observation=") + ) + end @test line(k) == "observation=$token" k2 = DataVault.keys(v)[2] mark_done!(v, k2) From 204bb654723516f4f257bb04b5b7f06dc3d37af5 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Fri, 25 Sep 2026 04:57:33 +0000 Subject: [PATCH 2/4] Keep the artifacts a JLL loaded, symlink targets, and the script that ran Found by recomputing a real study from its revision with an empty depot: JLD2 loads Zstd_jll, whose library lives in `artifacts/`, which no package tree holds; the library's `libzstd.so -> libzstd.so.1` links had no content to lay out; and `julia compute.jl` does not appear in `Base._included_files`, so nothing said which script to run. - Each artifact a loaded depot package's Artifacts.toml selects for this platform, where installed, is an `artifact::` root. - A symlink's target is kept as its blob. - Depot and artifact roots are kept whole, up to the hash limit, not only up to the 1 MiB meant for a study's own files. - `program` records `abspath(PROGRAM_FILE)`. Co-Authored-By: Claude Opus 5.5 (1M context) --- Project.toml | 2 ++ src/DataVault.jl | 1 + src/provenance/observe.jl | 48 +++++++++++++++++++++++++++++++++++++- test/vault/test_observe.jl | 17 ++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 19c8bdd..ae7b0d2 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ version = "0.8.7" authors = ["sota shimozono "] [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" @@ -17,6 +18,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [compat] Aqua = "0.8" +Artifacts = "1.11" CRC32c = "1.11" Dates = "1.11.0" JLD2 = "0.6.4" diff --git a/src/DataVault.jl b/src/DataVault.jl index d8fa7c3..7093dfc 100644 --- a/src/DataVault.jl +++ b/src/DataVault.jl @@ -29,6 +29,7 @@ using SHA using CRC32c: crc32c using Random: Random using LinearAlgebra: LinearAlgebra +using Artifacts: Artifacts using JSON3 using ParamIO diff --git a/src/provenance/observe.jl b/src/provenance/observe.jl index abfd845..4be86e4 100644 --- a/src/provenance/observe.jl +++ b/src/provenance/observe.jl @@ -117,9 +117,47 @@ function _depot_roots(existing) any(r -> _real(r.dir) == _real(dir) || _inside(dir, r.dir), existing) && continue push!(out, (name="pkg:$(id.name):$(id.uuid)", dir=dir, kind=:depot, tree=tree)) end + append!(out, _artifact_roots(out)) return sort!(out; by=r -> r.name) end +# The artifacts those packages' `Artifacts.toml` select for this platform, where installed: a JLL +# loads a library from `artifacts/`, which no package tree holds. Named by their tree hash, +# which is also their directory's name, so a restore knows where each goes and what it must hash to. +function _artifact_roots(packages) + out = Any[] + seen = Set{String}() + platform = Base.BinaryPlatforms.HostPlatform() + for p in packages + toml = joinpath(p.dir, "Artifacts.toml") + isfile(toml) || continue + dict = try + TOML.parsefile(toml) + catch + continue + end + for name in sort!(collect(keys(dict))) + meta = try + Artifacts.artifact_meta(name, dict, toml; platform) + catch + nothing + end + (meta === nothing || !haskey(meta, "git-tree-sha1")) && continue + tree = meta["git-tree-sha1"] + tree in seen && continue + dir = nothing + for d in DEPOT_PATH + cand = joinpath(d, "artifacts", tree) + isdir(cand) && (dir = cand; break) + end + dir === nothing && continue # lazy and never fetched + push!(seen, tree) + push!(out, (name="artifact:$name:$tree", dir=dir, kind=:artifact, tree=tree)) + end + end + return out +end + const _PathDep = NamedTuple{(:name, :uuid, :path),NTuple{3,String}} function _active_manifest()::Union{String,Nothing} @@ -195,8 +233,14 @@ end function _entry(root, rel, hash_limit, blobs, notes; materialize_limit)::SourceEntry full = joinpath(root.dir, rel) st = lstat(full) + # A package or artifact from a depot is kept whole: it is what only its origin could give + # back, and a library in it is routinely larger than any limit meant for a study's files. + root.kind in (:depot, :artifact) && (materialize_limit = hash_limit) if islink(st) target = readlink(full) + # The link's target is its content; kept, so that a library's `libx.so -> libx.so.1` + # can be laid out again. + blobs[bytes2hex(sha256(target))] = Vector{UInt8}(target) return SourceEntry( root.name, rel, @@ -417,7 +461,7 @@ function _root_record(root, loaded::String)::Dict{String,Any} root.dir, "status", "--porcelain", "--untracked-files=all", "--", "." ) record["dirty"] = porcelain === nothing ? "unknown" : string(!isempty(porcelain)) - elseif root.kind === :depot + elseif root.kind in (:depot, :artifact) # The tree the Manifest pins. Whether the directory still hashes to it is for a restore # to check: a depot is not written to after install, but nothing here proves it. record["head"] = root.tree @@ -550,6 +594,8 @@ function observe_sources( Dict{String,Any}(String(k) => v for (k, v) in process), ), "julia" => _julia_record(), + # The script `julia ` ran: not among `main_files`, which only lists includes. + "program" => isempty(PROGRAM_FILE) ? "" : abspath(PROGRAM_FILE), "env" => Dict{String,Any}(k => ENV[k] for k in ENV_RECORDED if haskey(ENV, k)), "environment" => _environment_record(vault), "revise_loaded" => revise, diff --git a/test/vault/test_observe.jl b/test/vault/test_observe.jl index b999444..96c5869 100644 --- a/test/vault/test_observe.jl +++ b/test/vault/test_observe.jl @@ -165,6 +165,23 @@ end end end +@testset "observe_sources: a symlink's target is kept, and so are the artifacts loaded" begin + with_observed_repo() do t + symlink("notes.dat", joinpath(t.repo, "link.dat")) + r = record(t, observe_sources(t.vault)) + blobs = joinpath(obs_dir(t), "sources", "blobs") + @test isfile(joinpath(blobs, bytes2hex(sha256("notes.dat")))) + @test read(joinpath(blobs, bytes2hex(sha256("notes.dat"))), String) == "notes.dat" + # Every artifact root is named by, and pinned to, its tree; and it is one that a loaded + # depot package's Artifacts.toml selects. + for a in (x for x in r["roots"] if x["kind"] == "artifact") + @test startswith(a["name"], "artifact:") && endswith(a["name"], ":" * a["head"]) + @test isdir(a["dir"]) && basename(a["dir"]) == a["head"] + end + @test haskey(r, "program") + end +end + @testset "observe_sources: which Julia binary, and BLAS's thread count" begin with_observed_repo() do t j = record(t, observe_sources(t.vault))["julia"] From 85fda76d7a98e92e5ae0ed9a34674717a04b16fe Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Fri, 25 Sep 2026 05:06:23 +0000 Subject: [PATCH 3/4] Format with JuliaFormatter 2, which CI runs Co-Authored-By: Claude Opus 5.5 (1M context) --- src/provenance/observe.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/provenance/observe.jl b/src/provenance/observe.jl index 4be86e4..f091f35 100644 --- a/src/provenance/observe.jl +++ b/src/provenance/observe.jl @@ -148,7 +148,7 @@ function _artifact_roots(packages) dir = nothing for d in DEPOT_PATH cand = joinpath(d, "artifacts", tree) - isdir(cand) && (dir = cand; break) + isdir(cand) && (dir=cand; break) end dir === nothing && continue # lazy and never fetched push!(seen, tree) From 3ad709998130f17c1472698b254a719807b40e35 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Fri, 25 Sep 2026 07:25:47 +0000 Subject: [PATCH 4/4] An artifact that cannot be resolved is said, not dropped From the review of this PR. `_artifact_roots` caught a TOML or `artifact_meta` failure and moved on, leaving the observation to look complete while a JLL's library was missing from it. It is now a note, and the inventory is then not complete, as a root that cannot be listed already was. Tests for the paths codecov found untested: an unreadable Artifacts.toml, a lazy artifact never fetched, a depot package kept whole under a 1-byte materialize limit, and `program` holding the script's path. The docstring now says what is kept for depot and artifact roots and for symlinks. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/provenance/observe.jl | 36 +++++++++++++++------ test/vault/test_observe.jl | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/src/provenance/observe.jl b/src/provenance/observe.jl index f091f35..f0db129 100644 --- a/src/provenance/observe.jl +++ b/src/provenance/observe.jl @@ -63,7 +63,7 @@ _inside(path, dir) = startswith(_real(path), rstrip(_real(dir), '/') * "/") # The study is the active environment's directory when the config lies inside it (a study that # carries its own Project.toml, so that a repository of many studies does not capture them all), # and otherwise the config's repository, or its directory outside git. -function _source_roots(vault::Vault; depot::Bool=false) +function _source_roots(vault::Vault; depot::Bool=false, notes=String[]) cfg = dirname(abspath(vault.config_path)) top = _git_read(cfg, "rev-parse", "--show-toplevel") project = Base.active_project() @@ -88,14 +88,14 @@ function _source_roots(vault::Vault; depot::Bool=false) ), ) end - depot && append!(roots, _depot_roots(roots)) + depot && append!(roots, _depot_roots(roots, notes)) return roots end # Every package this process loaded from a depot (`packages//`), with the tree hash the # active Manifest pins it to. These are what a registered or git-URL dependency is: a tree that # only a registry, a package server or a git remote can give back, so the snapshot keeps it. -function _depot_roots(existing) +function _depot_roots(existing, notes=String[]) manifest = _active_manifest() manifest === nothing && return Any[] pinned = Dict{String,String}() @@ -117,14 +117,16 @@ function _depot_roots(existing) any(r -> _real(r.dir) == _real(dir) || _inside(dir, r.dir), existing) && continue push!(out, (name="pkg:$(id.name):$(id.uuid)", dir=dir, kind=:depot, tree=tree)) end - append!(out, _artifact_roots(out)) + append!(out, _artifact_roots(out, notes)) return sort!(out; by=r -> r.name) end # The artifacts those packages' `Artifacts.toml` select for this platform, where installed: a JLL # loads a library from `artifacts/`, which no package tree holds. Named by their tree hash, # which is also their directory's name, so a restore knows where each goes and what it must hash to. -function _artifact_roots(packages) +# One that cannot be resolved is a note, and the inventory is then not complete: a recomputation +# would find the library missing, and the observation must not look whole. +function _artifact_roots(packages, notes=String[]) out = Any[] seen = Set{String}() platform = Base.BinaryPlatforms.HostPlatform() @@ -133,13 +135,21 @@ function _artifact_roots(packages) isfile(toml) || continue dict = try TOML.parsefile(toml) - catch + catch e + push!( + notes, + "$(p.name): Artifacts.toml could not be resolved: $(sprint(showerror, e))", + ) continue end for name in sort!(collect(keys(dict))) meta = try Artifacts.artifact_meta(name, dict, toml; platform) - catch + catch e + push!( + notes, + "$(p.name): artifact $name could not be resolved: $(sprint(showerror, e))", + ) nothing end (meta === nothing || !haskey(meta, "git-tree-sha1")) && continue @@ -530,7 +540,11 @@ as a worker id), which snapshot (`source`), each root's git HEAD and whether it build (with the binary's digest, the platform and BLAS's thread count) and a fixed list of environment variables, and the **binding**: how far the code this process has loaded was checked against the snapshot (see [`binding_of`](@ref)). File contents are stored for `.jl` and `.toml` -files and for any file up to `materialize_limit`; every file is inventoried by size and digest. +files and for any other file up to `materialize_limit`, and for every file of a `depot` or +`artifact` root up to `hash_limit` (a library there is routinely larger); a symlink's target is kept +as its content. Every file is inventoried by size, and by digest up to `hash_limit`. An artifact +that cannot be resolved, like a root that cannot be listed, is a note and leaves the inventory +incomplete. Depot packages are kept at `run-start` only by default: that is the process that computed, and a render's plotting stack is large and not what a recomputation needs. @@ -545,13 +559,15 @@ function observe_sources( )::String _refuse_if_readonly(vault, "observe_sources") observed_at = _utc_stamp() - roots = _source_roots(vault; depot=depot_packages) + root_notes = String[] + roots = _source_roots(vault; depot=depot_packages, notes=root_notes) entries, blobs, notes = _inventory( roots; hash_limit, materialize_limit, exclude=vault.outdir ) + notes = vcat(root_notes, notes) complete = !any(e -> e.sha256 == "skipped" || e.type == "dir", entries) && - !any(n -> occursin("could not be listed", n), notes) + !any(n -> occursin(r"could not be (listed|resolved)", n), notes) state = Dict{String,Any}( "recipe" => SOURCE_RECIPE, "roots" => [ diff --git a/test/vault/test_observe.jl b/test/vault/test_observe.jl index 96c5869..36d16dc 100644 --- a/test/vault/test_observe.jl +++ b/test/vault/test_observe.jl @@ -182,6 +182,70 @@ end end end +@testset "observe_sources: an artifact that cannot be resolved is said, not dropped" begin + pkg = mktempdir() + try + fake = ( + name="pkg:Fake:00000000-0000-0000-0000-000000000000", + dir=pkg, + kind=:depot, + tree="", + ) + # Unreadable: a note, which the observation turns into an incomplete inventory. + write(joinpath(pkg, "Artifacts.toml"), "this is [[ not toml") + notes = String[] + @test isempty(DataVault._artifact_roots([fake], notes)) + @test any(n -> occursin("Artifacts.toml could not be resolved", n), notes) + # Readable but never fetched (lazy): nothing to keep, and nothing wrong either. + write(joinpath(pkg, "Artifacts.toml"), "[foo]\ngit-tree-sha1 = \"$("0"^40)\"\n") + notes = String[] + @test isempty(DataVault._artifact_roots([fake], notes)) && isempty(notes) + finally + rm(pkg; recursive=true) + end +end + +@testset "observe_sources: a depot package is kept whole, whatever the size limit" begin + with_observed_repo() do t + uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" # JLD2, loaded from a depot + r = record(t, observe_sources(t.vault; materialize_limit=1)) + snap = joinpath(obs_dir(t), "sources", r["source"]) + rows = [ + split(l, '\t') for + l in eachline(joinpath(snap, "files.tsv")) if startswith(l, "pkg:JLD2:$uuid\t") + ] + if isempty(rows) + @test_broken !isempty(rows) # JLD2 not from a depot here + else + # A file that is neither .jl nor .toml and larger than the 1-byte limit. + other = [ + x for x in rows if x[3] == "file" && + parse(Int, x[5]) > 1 && + !endswith(x[2], ".jl") && + !endswith(x[2], ".toml") && + x[6] != "skipped" + ] + @test !isempty(other) + @test all(x -> isfile(joinpath(obs_dir(t), "sources", "blobs", x[6])), other) + end + end +end + +@testset "observe_sources: `program` is the script `julia ` ran" begin + with_observed_repo() do t + script = joinpath(t.repo, "run.jl") + before = PROGRAM_FILE + @eval Base PROGRAM_FILE = $script + try + @test record(t, observe_sources(t.vault))["program"] == abspath(script) + finally + @eval Base PROGRAM_FILE = $before + end + @test record(t, observe_sources(t.vault))["program"] == + (isempty(before) ? "" : abspath(before)) + end +end + @testset "observe_sources: which Julia binary, and BLAS's thread count" begin with_observed_repo() do t j = record(t, observe_sources(t.vault))["julia"]