diff --git a/Project.toml b/Project.toml index 526c7d2..ae7b0d2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,13 +1,15 @@ name = "DataVault" uuid = "23f5f8f6-b4da-40ee-8c72-c53b6c5de94f" -version = "0.8.6" +version = "0.8.7" authors = ["sota shimozono "] [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ParamIO = "938a3ac2-d340-473c-bcf1-88af577e4ccf" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -16,10 +18,12 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [compat] Aqua = "0.8" +Artifacts = "1.11" CRC32c = "1.11" Dates = "1.11.0" JLD2 = "0.6.4" JSON3 = "1" +LinearAlgebra = "1.11" Logging = "1.11" # 0.4.11 is the floor, not a preference: `artifact_identity` / `artifact_key` (what `artifact!` # stores by) arrived there; `float_format` and the 3-arg `format_path` arrived in 0.4.0. diff --git a/src/DataVault.jl b/src/DataVault.jl index 29111fa..7093dfc 100644 --- a/src/DataVault.jl +++ b/src/DataVault.jl @@ -28,6 +28,8 @@ using JLD2, TOML, Dates, Printf using SHA using CRC32c: crc32c using Random: Random +using LinearAlgebra: LinearAlgebra +using Artifacts: Artifacts using JSON3 using ParamIO diff --git a/src/provenance/observe.jl b/src/provenance/observe.jl index 98fb0a0..f0db129 100644 --- a/src/provenance/observe.jl +++ b/src/provenance/observe.jl @@ -16,7 +16,8 @@ const SOURCE_RECIPE = "src1" const OBSERVATION_VERSION = 1 -const MATERIALIZE_EXTENSIONS = (".jl", ".toml") # contents kept; every other file inventoried only +const MATERIALIZE_EXTENSIONS = (".jl", ".toml") # contents always kept, at any size up to the hash limit +const DEFAULT_MATERIALIZE_LIMIT = 1024^2 # any other file: contents kept up to this size const DEFAULT_HASH_LIMIT = 64 * 1024^2 # larger files are inventoried without a digest const ENV_RECORDED = ( "JULIA_NUM_THREADS", @@ -55,30 +56,118 @@ _inside(path, dir) = startswith(_real(path), rstrip(_real(dir), '/') * "/") # ── roots ───────────────────────────────────────────────────────────────────────────────────── -# The config's repository (or directory), and every `path` dependency of the active environment -# that is not already inside it. Names are logical — no absolute path enters the snapshot. -function _source_roots(vault::Vault) +# The study, every `path` dependency of the active environment that is not already inside it, and — +# when `depot` — every package this process loaded from a depot. Names are logical: no absolute +# path enters the snapshot. +# +# The study is the active environment's directory when the config lies inside it (a study that +# carries its own Project.toml, so that a repository of many studies does not capture them all), +# and otherwise the config's repository, or its directory outside git. +function _source_roots(vault::Vault; depot::Bool=false, notes=String[]) cfg = dirname(abspath(vault.config_path)) top = _git_read(cfg, "rev-parse", "--show-toplevel") - roots = [ - if top === nothing - (name="config", dir=cfg, kind=:plain) - else - (name="config", dir=top, kind=:git) - end, - ] + project = Base.active_project() + envdir = project === nothing ? nothing : dirname(abspath(project)) + study = if envdir !== nothing && (_inside(cfg, envdir) || _real(cfg) == _real(envdir)) + envdir + else + something(top, cfg) + end + roots = Any[(name="config", dir=study, kind=top === nothing ? :plain : :git, tree="")] for dep in _path_dependencies() any(r -> _inside(dep.path, r.dir) || _real(dep.path) == _real(r.dir), roots) && continue ingit = _git_read(dep.path, "rev-parse", "--show-toplevel") !== nothing push!( roots, - (name="pkg:$(dep.name):$(dep.uuid)", dir=dep.path, kind=ingit ? :git : :plain), + ( + name="pkg:$(dep.name):$(dep.uuid)", + dir=dep.path, + kind=ingit ? :git : :plain, + tree="", + ), ) end + depot && append!(roots, _depot_roots(roots, notes)) return roots end +# Every package this process loaded from a depot (`packages//`), with the tree hash the +# active Manifest pins it to. These are what a registered or git-URL dependency is: a tree that +# only a registry, a package server or a git remote can give back, so the snapshot keeps it. +function _depot_roots(existing, notes=String[]) + manifest = _active_manifest() + manifest === nothing && return Any[] + pinned = Dict{String,String}() + for (_, entries) in get(TOML.parsefile(manifest), "deps", Dict{String,Any}()), + e in entries + + haskey(e, "git-tree-sha1") && + haskey(e, "uuid") && + (pinned[e["uuid"]] = e["git-tree-sha1"]) + end + depots = [joinpath(_real(d), "packages") for d in DEPOT_PATH if isdir(d)] + out = Any[] + for (id, origin) in Base.pkgorigins + (id.uuid === nothing || origin.path === nothing) && continue + tree = get(pinned, string(id.uuid), nothing) + tree === nothing && continue + any(d -> _inside(origin.path, d), depots) || continue + dir = dirname(dirname(origin.path)) # /src/.jl → + any(r -> _real(r.dir) == _real(dir) || _inside(dir, r.dir), existing) && continue + push!(out, (name="pkg:$(id.name):$(id.uuid)", dir=dir, kind=:depot, tree=tree)) + end + append!(out, _artifact_roots(out, notes)) + return sort!(out; by=r -> r.name) +end + +# The artifacts those packages' `Artifacts.toml` select for this platform, where installed: a JLL +# loads a library from `artifacts/`, which no package tree holds. Named by their tree hash, +# which is also their directory's name, so a restore knows where each goes and what it must hash to. +# One that cannot be resolved is a note, and the inventory is then not complete: a recomputation +# would find the library missing, and the observation must not look whole. +function _artifact_roots(packages, notes=String[]) + out = Any[] + seen = Set{String}() + platform = Base.BinaryPlatforms.HostPlatform() + for p in packages + toml = joinpath(p.dir, "Artifacts.toml") + isfile(toml) || continue + dict = try + TOML.parsefile(toml) + catch e + push!( + notes, + "$(p.name): Artifacts.toml could not be resolved: $(sprint(showerror, e))", + ) + continue + end + for name in sort!(collect(keys(dict))) + meta = try + Artifacts.artifact_meta(name, dict, toml; platform) + catch e + push!( + notes, + "$(p.name): artifact $name could not be resolved: $(sprint(showerror, e))", + ) + nothing + end + (meta === nothing || !haskey(meta, "git-tree-sha1")) && continue + tree = meta["git-tree-sha1"] + tree in seen && continue + dir = nothing + for d in DEPOT_PATH + cand = joinpath(d, "artifacts", tree) + isdir(cand) && (dir=cand; break) + end + dir === nothing && continue # lazy and never fetched + push!(seen, tree) + push!(out, (name="artifact:$name:$tree", dir=dir, kind=:artifact, tree=tree)) + end + end + return out +end + const _PathDep = NamedTuple{(:name, :uuid, :path),NTuple{3,String}} function _active_manifest()::Union{String,Nothing} @@ -108,20 +197,28 @@ end # ── inventory ───────────────────────────────────────────────────────────────────────────────── -function _root_files(root)::Union{Vector{String},Nothing} +# Relative to the root. Anything under `exclude` (the vault's own output) is not source, even when +# it sits inside the study and nothing ignores it. +function _root_files(root; exclude=nothing)::Union{Vector{String},Nothing} + keep(rel) = exclude === nothing || !_inside(joinpath(root.dir, rel), exclude) if root.kind === :git out = _git_read( root.dir, "ls-files", "-z", "--cached", "--others", "--exclude-standard" ) out === nothing && return nothing - return sort!(unique!(filter!(!isempty, String.(split(out, '\0'))))) + return sort!(filter!(keep, unique!(filter!(!isempty, String.(split(out, '\0')))))) end files = String[] for (dir, dirs, fs) in walkdir(root.dir) filter!(d -> d != ".git", dirs) + # Do not walk into the output at all: it can be far larger than the study. + exclude === nothing || filter!(dirs) do d + p = joinpath(dir, d) + return !(_real(p) == _real(exclude) || _inside(p, exclude)) + end append!(files, relpath(joinpath(dir, f), root.dir) for f in fs) end - return sort!(files) + return sort!(filter!(keep, files)) end struct SourceEntry @@ -135,11 +232,25 @@ struct SourceEntry full::String end -function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry +# A file's contents are kept when it is `.jl`/`.toml` (at any size up to the hash limit) or no +# larger than `materialize_limit`: the snapshot must hold what a package reads besides its code (a +# table, a template, a script), and an extension list drops those without a word. +function _materialize(rel, size, limit) + return size <= limit || + any(ext -> endswith(lowercase(rel), ext), MATERIALIZE_EXTENSIONS) +end + +function _entry(root, rel, hash_limit, blobs, notes; materialize_limit)::SourceEntry full = joinpath(root.dir, rel) st = lstat(full) + # A package or artifact from a depot is kept whole: it is what only its origin could give + # back, and a library in it is routinely larger than any limit meant for a study's files. + root.kind in (:depot, :artifact) && (materialize_limit = hash_limit) if islink(st) target = readlink(full) + # The link's target is its content; kept, so that a library's `libx.so -> libx.so.1` + # can be laid out again. + blobs[bytes2hex(sha256(target))] = Vector{UInt8}(target) return SourceEntry( root.name, rel, @@ -160,8 +271,7 @@ function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry end bytes = read(full) sha = bytes2hex(sha256(bytes)) - any(ext -> endswith(lowercase(rel), ext), MATERIALIZE_EXTENSIONS) && - (blobs[sha] = bytes) + _materialize(rel, length(bytes), materialize_limit) && (blobs[sha] = bytes) return SourceEntry( root.name, rel, "file", mode, length(bytes), sha, crc32c(bytes), full ) @@ -172,17 +282,22 @@ function _entry(root, rel, hash_limit, blobs, notes)::SourceEntry return SourceEntry(root.name, rel, "missing", "-", 0, "", 0x00000000, full) end -function _inventory(roots; hash_limit::Integer) +function _inventory( + roots; hash_limit::Integer, materialize_limit::Integer=0, exclude=nothing +) entries = SourceEntry[] blobs = Dict{String,Vector{UInt8}}() notes = String[] for root in roots - files = _root_files(root) + files = _root_files(root; exclude) if files === nothing push!(notes, "$(root.name): files could not be listed") continue end - append!(entries, _entry(root, rel, hash_limit, blobs, notes) for rel in files) + append!( + entries, + _entry(root, rel, hash_limit, blobs, notes; materialize_limit) for rel in files + ) end return entries, blobs, notes end @@ -352,8 +467,16 @@ function _root_record(root, loaded::String)::Dict{String,Any} observed = _git_observe(root.dir) record["head"] = observed.commit record["object_format"] = observed.object_format - porcelain = _git_read(root.dir, "status", "--porcelain", "--untracked-files=all") + porcelain = _git_read( + root.dir, "status", "--porcelain", "--untracked-files=all", "--", "." + ) record["dirty"] = porcelain === nothing ? "unknown" : string(!isempty(porcelain)) + elseif root.kind in (:depot, :artifact) + # The tree the Manifest pins. Whether the directory still hashes to it is for a restore + # to check: a depot is not written to after install, but nothing here proves it. + record["head"] = root.tree + record["object_format"] = "sha1" + record["dirty"] = "unknown" else record["head"] = record["object_format"] = record["dirty"] = "unknown" end @@ -376,7 +499,8 @@ end function _julia_record()::Dict{String,Any} opts = Base.JLOptions() - return Dict{String,Any}( + exe = joinpath(Sys.BINDIR, Base.julia_exename()) + record = Dict{String,Any}( "version" => string(VERSION), "commit" => Base.GIT_VERSION_INFO.commit, "image_file" => opts.image_file == C_NULL ? "" : unsafe_string(opts.image_file), @@ -384,40 +508,77 @@ function _julia_record()::Dict{String,Any} "opt_level" => Int(opts.opt_level), "fast_math" => Int(opts.fast_math), "threads" => Threads.nthreads(), + # Which binary, not only which version: a launcher (juliaup) given the same command in + # another HOME starts another Julia. The digest is of the binary itself. + "bindir" => Sys.BINDIR, + "platform" => Base.BinaryPlatforms.triplet(Base.BinaryPlatforms.HostPlatform()), + "cpu_name" => Sys.CPU_NAME, + # BLAS's own thread count, whatever set it: results differ in the last bits between + # counts, so a bitwise comparison is only meaningful at the one recorded here. + "blas_threads" => LinearAlgebra.BLAS.get_num_threads(), + "blas_libraries" => + [basename(l.libname) for l in LinearAlgebra.BLAS.get_config().loaded_libs], ) + isfile(exe) && (record["executable_sha256"] = bytes2hex(open(sha256, exe))) + return record end """ - observe_sources(vault; phase = "run-start", process = Dict(), hash_limit = 64 MiB) -> token + observe_sources(vault; phase = "run-start", process = Dict(), hash_limit = 64 MiB, + materialize_limit = 1 MiB, depot_packages = (phase == "run-start")) -> token -Observe the source roots this process can see — the config's repository and every `path` -dependency of the active environment — store the snapshot (once per distinct content) and an -observation record, and return the record's token for [`mark_done!`](@ref)'s `observation`. +Observe the source roots this process can see, store the snapshot (once per distinct content) and +an observation record, and return the record's token for [`mark_done!`](@ref)'s `observation`. + +The roots are the study (the active environment's directory when the config lies inside it, +otherwise the config's repository), every `path` dependency of the active environment, and, when +`depot_packages`, every package this process loaded from a depot, recorded as a `depot` root whose +`head` is the tree hash the Manifest pins. The vault's own output directory is never source. The record says when (`observed_at`, `phase`), where (host, pid, and whatever `process` adds, such as a worker id), which snapshot (`source`), each root's git HEAD and whether it was dirty, the Julia -build and a fixed list of environment variables, and the **binding**: how far the code this process -has loaded was checked against the snapshot (see [`binding_of`](@ref)). File contents are stored -only for `.jl` and `.toml` files; every other file is inventoried by size and digest. +build (with the binary's digest, the platform and BLAS's thread count) and a fixed list of +environment variables, and the **binding**: how far the code this process has loaded was checked +against the snapshot (see [`binding_of`](@ref)). File contents are stored for `.jl` and `.toml` +files and for any other file up to `materialize_limit`, and for every file of a `depot` or +`artifact` root up to `hash_limit` (a library there is routinely larger); a symlink's target is kept +as its content. Every file is inventoried by size, and by digest up to `hash_limit`. An artifact +that cannot be resolved, like a root that cannot be listed, is a note and leaves the inventory +incomplete. + +Depot packages are kept at `run-start` only by default: that is the process that computed, and a +render's plotting stack is large and not what a recomputation needs. """ function observe_sources( vault::Vault; phase::AbstractString="run-start", process::AbstractDict=Dict{String,Any}(), hash_limit::Integer=DEFAULT_HASH_LIMIT, + materialize_limit::Integer=DEFAULT_MATERIALIZE_LIMIT, + depot_packages::Bool=(phase == "run-start"), )::String _refuse_if_readonly(vault, "observe_sources") observed_at = _utc_stamp() - roots = _source_roots(vault) - entries, blobs, notes = _inventory(roots; hash_limit) + root_notes = String[] + roots = _source_roots(vault; depot=depot_packages, notes=root_notes) + entries, blobs, notes = _inventory( + roots; hash_limit, materialize_limit, exclude=vault.outdir + ) + notes = vcat(root_notes, notes) complete = !any(e -> e.sha256 == "skipped" || e.type == "dir", entries) && - !any(n -> occursin("could not be listed", n), notes) + !any(n -> occursin(r"could not be (listed|resolved)", n), notes) state = Dict{String,Any}( "recipe" => SOURCE_RECIPE, - "roots" => [Dict("name" => r.name, "kind" => String(r.kind)) for r in roots], + "roots" => [ + merge( + Dict("name" => r.name, "kind" => String(r.kind)), + isempty(r.tree) ? Dict{String,String}() : Dict("tree" => r.tree), + ) for r in roots + ], "inventory_complete" => complete, "materialized" => collect(MATERIALIZE_EXTENSIONS), + "materialize_limit" => materialize_limit, "notes" => notes, ) source = _publish_snapshot(vault, _files_tsv(entries), state, blobs) @@ -449,6 +610,8 @@ function observe_sources( Dict{String,Any}(String(k) => v for (k, v) in process), ), "julia" => _julia_record(), + # The script `julia ` ran: not among `main_files`, which only lists includes. + "program" => isempty(PROGRAM_FILE) ? "" : abspath(PROGRAM_FILE), "env" => Dict{String,Any}(k => ENV[k] for k in ENV_RECORDED if haskey(ENV, k)), "environment" => _environment_record(vault), "revise_loaded" => revise, diff --git a/test/vault/test_observe.jl b/test/vault/test_observe.jl index 473beda..36d16dc 100644 --- a/test/vault/test_observe.jl +++ b/test/vault/test_observe.jl @@ -67,13 +67,192 @@ config_root(rec) = only(r for r in rec["roots"] if r["name"] == "config") end end -@testset "observe_sources: only .jl and .toml contents are kept" begin +@testset "observe_sources: contents are kept by size, and .jl/.toml at any size" begin + sha(file) = bytes2hex(sha256(read(file))) with_observed_repo() do t observe_sources(t.vault) blobs = joinpath(obs_dir(t), "sources", "blobs") - sha(file) = bytes2hex(sha256(read(file))) + @test isfile(joinpath(blobs, sha(joinpath(t.pkg, "src", "$(t.name).jl")))) + # A data file a package reads is source too; an extension list would drop it silently. + @test isfile(joinpath(blobs, sha(joinpath(t.repo, "notes.dat")))) + end + with_observed_repo() do t + r = record(t, observe_sources(t.vault; materialize_limit=4)) + blobs = joinpath(obs_dir(t), "sources", "blobs") @test isfile(joinpath(blobs, sha(joinpath(t.pkg, "src", "$(t.name).jl")))) @test !isfile(joinpath(blobs, sha(joinpath(t.repo, "notes.dat")))) + state = TOML.parsefile(joinpath(obs_dir(t), "sources", r["source"], "state.toml")) + @test state["materialize_limit"] == 4 + end +end + +@testset "observe_sources: the vault's own output inside the study is not source" begin + repo = mktempdir() + try + cp(_OBS_CFG, joinpath(repo, "study.toml")) + out = joinpath(repo, "out") # inside the study, ignored by nothing + v = Vault(joinpath(repo, "study.toml"); outdir=out) + k = DataVault.keys(v)[1] + mark_done!(v, k; result=DataVault.save!(v, k, Dict("x" => 1.0))) + store = joinpath(out, ".datavault", "test_study") + r = TOML.parsefile(joinpath(store, "observations", "$(observe_sources(v)).toml")) + tsv = read(joinpath(store, "sources", r["source"], "files.tsv"), String) + @test occursin("config\tstudy.toml\t", tsv) + @test !occursin("config\tout/", tsv) + finally + rm(repo; recursive=true, force=true) + end +end + +@testset "observe_sources: a study with its own environment is its own root" begin + with_observed_repo() do t + study = joinpath(t.repo, "studies", "one") + mkpath(study) + cp(_OBS_CFG, joinpath(study, "study.toml")) + write(joinpath(study, "Project.toml"), "[deps]\n") + write(joinpath(t.repo, "studies", "other.jl"), "x = 1\n") # a neighbour, not this study + git!(t.repo, "add", "-A") + git!(t.repo, "commit", "-qm", "studies") + before = Base.active_project() + Base.set_active_project(joinpath(study, "Project.toml")) + try + v = Vault(joinpath(study, "study.toml"); outdir=t.out) + r = record(t, observe_sources(v)) + tsv = read(joinpath(obs_dir(t), "sources", r["source"], "files.tsv"), String) + @test occursin("config\tstudy.toml\t", tsv) + @test occursin("config\tProject.toml\t", tsv) + @test !occursin("other.jl", tsv) && !occursin("notes.dat", tsv) + c = config_root(r) + @test c["kind"] == "git" + @test c["head"] == readchomp(`git -C $(t.repo) rev-parse HEAD`) + @test c["dirty"] == "false" + write(joinpath(t.repo, "notes.dat"), "changed outside the study") + @test config_root(record(t, observe_sources(v)))["dirty"] == "false" + finally + Base.set_active_project(before) + end + end +end + +@testset "observe_sources: the packages a computing process loaded are kept" begin + with_observed_repo() do t + manifest = DataVault._active_manifest() + pinned = Dict{String,String}() + if manifest !== nothing + for (_, es) in TOML.parsefile(manifest)["deps"], e in es + haskey(e, "git-tree-sha1") && (pinned[e["uuid"]] = e["git-tree-sha1"]) + end + end + # JLD2 is loaded by DataVault itself, from a depot, whenever these tests run. + uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" + r = record(t, observe_sources(t.vault)) + if haskey(pinned, uuid) + j = only(x for x in r["roots"] if x["name"] == "pkg:JLD2:$uuid") + @test j["kind"] == "depot" && j["head"] == pinned[uuid] + snap = joinpath(obs_dir(t), "sources", r["source"]) + @test occursin( + "pkg:JLD2:$uuid\tsrc/JLD2.jl\tfile", + read(joinpath(snap, "files.tsv"), String), + ) + state = TOML.parsefile(joinpath(snap, "state.toml")) + @test any(x -> get(x, "tree", "") == j["head"], state["roots"]) + else + @test_broken haskey(pinned, uuid) # JLD2 is not pinned by a tree here + end + # A render is not the computing process: its plotting stack is not kept by default. + rr = record(t, observe_sources(t.vault; phase="render")) + @test !any(x -> x["kind"] == "depot", rr["roots"]) + end +end + +@testset "observe_sources: a symlink's target is kept, and so are the artifacts loaded" begin + with_observed_repo() do t + symlink("notes.dat", joinpath(t.repo, "link.dat")) + r = record(t, observe_sources(t.vault)) + blobs = joinpath(obs_dir(t), "sources", "blobs") + @test isfile(joinpath(blobs, bytes2hex(sha256("notes.dat")))) + @test read(joinpath(blobs, bytes2hex(sha256("notes.dat"))), String) == "notes.dat" + # Every artifact root is named by, and pinned to, its tree; and it is one that a loaded + # depot package's Artifacts.toml selects. + for a in (x for x in r["roots"] if x["kind"] == "artifact") + @test startswith(a["name"], "artifact:") && endswith(a["name"], ":" * a["head"]) + @test isdir(a["dir"]) && basename(a["dir"]) == a["head"] + end + @test haskey(r, "program") + end +end + +@testset "observe_sources: an artifact that cannot be resolved is said, not dropped" begin + pkg = mktempdir() + try + fake = ( + name="pkg:Fake:00000000-0000-0000-0000-000000000000", + dir=pkg, + kind=:depot, + tree="", + ) + # Unreadable: a note, which the observation turns into an incomplete inventory. + write(joinpath(pkg, "Artifacts.toml"), "this is [[ not toml") + notes = String[] + @test isempty(DataVault._artifact_roots([fake], notes)) + @test any(n -> occursin("Artifacts.toml could not be resolved", n), notes) + # Readable but never fetched (lazy): nothing to keep, and nothing wrong either. + write(joinpath(pkg, "Artifacts.toml"), "[foo]\ngit-tree-sha1 = \"$("0"^40)\"\n") + notes = String[] + @test isempty(DataVault._artifact_roots([fake], notes)) && isempty(notes) + finally + rm(pkg; recursive=true) + end +end + +@testset "observe_sources: a depot package is kept whole, whatever the size limit" begin + with_observed_repo() do t + uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" # JLD2, loaded from a depot + r = record(t, observe_sources(t.vault; materialize_limit=1)) + snap = joinpath(obs_dir(t), "sources", r["source"]) + rows = [ + split(l, '\t') for + l in eachline(joinpath(snap, "files.tsv")) if startswith(l, "pkg:JLD2:$uuid\t") + ] + if isempty(rows) + @test_broken !isempty(rows) # JLD2 not from a depot here + else + # A file that is neither .jl nor .toml and larger than the 1-byte limit. + other = [ + x for x in rows if x[3] == "file" && + parse(Int, x[5]) > 1 && + !endswith(x[2], ".jl") && + !endswith(x[2], ".toml") && + x[6] != "skipped" + ] + @test !isempty(other) + @test all(x -> isfile(joinpath(obs_dir(t), "sources", "blobs", x[6])), other) + end + end +end + +@testset "observe_sources: `program` is the script `julia ` ran" begin + with_observed_repo() do t + script = joinpath(t.repo, "run.jl") + before = PROGRAM_FILE + @eval Base PROGRAM_FILE = $script + try + @test record(t, observe_sources(t.vault))["program"] == abspath(script) + finally + @eval Base PROGRAM_FILE = $before + end + @test record(t, observe_sources(t.vault))["program"] == + (isempty(before) ? "" : abspath(before)) + end +end + +@testset "observe_sources: which Julia binary, and BLAS's thread count" begin + with_observed_repo() do t + j = record(t, observe_sources(t.vault))["julia"] + @test j["blas_threads"] == DataVault.LinearAlgebra.BLAS.get_num_threads() + @test occursin(r"^[0-9a-f]{64}$", j["executable_sha256"]) + @test j["bindir"] == Sys.BINDIR + @test !isempty(j["platform"]) && !isempty(j["blas_libraries"]) end end @@ -147,10 +326,12 @@ end k = DataVault.keys(v)[1] token = observe_sources(v) mark_done!(v, k; result=DataVault.save!(v, k, Dict("x" => 1.0)), observation=token) - line(key) = only( - l for - l in eachline(DataVault._done_file(v, key)) if startswith(l, "observation=") - ) + function line(key) + return only( + l for + l in eachline(DataVault._done_file(v, key)) if startswith(l, "observation=") + ) + end @test line(k) == "observation=$token" k2 = DataVault.keys(v)[2] mark_done!(v, k2)