diff --git a/build.zig b/build.zig index 16c06d45..81798bce 100644 --- a/build.zig +++ b/build.zig @@ -484,7 +484,16 @@ pub fn build(b: *std.Build) void { // All pure packages, imported by name into engine / libtile57.a / the baker. // (portray is libc, wired separately into the lib + baker only.) + // Charts read straight out of a .zip, and the text/pictures a cell points + // at. Both are pure std and both are needed by the engine root AND by the + // separately-compiled chart module, so they must be modules: a relative + // import from each would put one file in two modules. + const zipsrc_mod = b.addModule("zipsrc", .{ .root_source_file = b.path("src/zipsrc.zig") }); + const auxfiles_mod = b.addModule("auxfiles", .{ .root_source_file = b.path("src/auxfiles.zig") }); + const pure_pkgs = [_]std.Build.Module.Import{ + .{ .name = "zipsrc", .module = zipsrc_mod }, + .{ .name = "auxfiles", .module = auxfiles_mod }, .{ .name = "s57", .module = s57_mod }, .{ .name = "s101", .module = s101_mod }, .{ .name = "tiles", .module = tiles_mod }, @@ -681,6 +690,8 @@ pub fn build(b: *std.Build) void { .{ .name = "portray", .module = portray_mod }, .{ .name = "sprite", .module = sprite_mod }, .{ .name = "catalog", .module = catalog_embed }, + .{ .name = "zipsrc", .module = zipsrc_mod }, + .{ .name = "auxfiles", .module = auxfiles_mod }, }, }); chart_mod.addImport("style", style_mod); // linestyle XML analysis @@ -837,6 +848,10 @@ pub fn build(b: *std.Build) void { }); _ = addPkgTest(b, test_step, "src/style/style.zig", target, optimize, &.{}); _ = addPkgTest(b, test_step, "src/errors.zig", target, optimize, &.{}); + // Charts read straight out of a .zip, and the aux files that travel with + // them: pure over std, so each tests alone. + _ = addPkgTest(b, test_step, "src/zipsrc.zig", target, optimize, &.{}); + _ = addPkgTest(b, test_step, "src/auxfiles.zig", target, optimize, &.{}); // Geometry core for the cross-band composition. No longer std-only: plane.zig // reads its partition tuning/stats valves via std.c.getenv, so the test binary // needs libc for the same reason compose's does, below. diff --git a/include/tile57.h b/include/tile57.h index 43f8a1a9..67b02243 100644 --- a/include/tile57.h +++ b/include/tile57.h @@ -290,6 +290,77 @@ tile57_status tile57_bake_rasters(const char *const *in_paths, const char *const tile57_bake_progress progress, tile57_bake_label label, void *progress_ctx, uint32_t *out_baked, tile57_error *err); +/* ---- charts inside a .zip --------------------------------------------------- + * + * A chart archive arrives as one download and is mostly not the file you want: + * NOAA's All_ENCs.zip is 788 MB of deflate holding 2.0 GiB across 27,680 + * entries. Unzipping it costs the mariner 2.0 GiB of disk that is dead the + * moment the import ends, on top of the charts they keep. These calls read the + * charts out of the archive where they lie: each cell is inflated when its turn + * comes, baked, and freed, so peak memory is the worker count times one chart + * and no intermediate copy is ever written. + * + * Every call opens the archive, works, and closes it. Walking the central + * directory of that 27,680-entry archive costs about 8 ms, so there is no + * handle to hold, close, or keep off another thread. + * + * `names` are entry names exactly as tile57_zip_list reports them. Destinations + * are always the CALLER's `out_paths`, never a name from the archive, so an + * entry called "../../etc/rc" cannot choose where anything lands. */ + +/* List what a .zip holds, into *out / *out_len (free with tile57_free), as + * [{"name":..,"size":..,"packed":..}, ..] in central-directory order. Sizes are + * uncompressed and compressed bytes. Directory entries are omitted. The buffer + * is NUL-terminated past *out_len, so it reads as a C string too. + * + * This says what is IN the archive, not what is a chart: classification is the + * host's, and only the host knows which formats it supports. */ +tile57_status tile57_zip_list(const char *zip_path, uint8_t **out, size_t *out_len, + tile57_error *err); + +/* tile57_bake_files, reading each cell straight out of `zip_path`: names[i] is + * a .000 entry, out_paths[i] is where its PMTiles goes. Its .001.. updates are + * found and applied from the archive, by the same rule the on-disk bake uses on + * a cell's directory: sequential from .001, stopping at the first gap. + * + * The text and pictures a cell references (TXTDSC, PICREP) are written beside + * the archive with an index.json manifest, so a pick report can resolve them + * through tile57_aux_open — but ONLY when out_paths[i] puts the archive in a + * directory named for the chart (/US1EEZ3M/US1EEZ3M.pmtiles). Those files + * are named per exchange set rather than per chart, so charts baked flat into + * one directory would overwrite each other's manifests; rather than guess, the + * rule is the exchange set's own shape. Same contract otherwise — caller-owned + * lists, `label` names a finished chart by index, `progress` may cancel, + * *out_baked counts what was written. A name the archive does not hold is + * skipped, like a cell that fails to bake. */ +tile57_status tile57_bake_zip_charts(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + +/* tile57_bake_rasters, reading each BSB/KAP sheet straight out of `zip_path`. + * Same contract as tile57_bake_rasters, including that `workers` is a memory + * bound: a sheet is decoded whole whether it came from a file or an archive. */ +tile57_status tile57_bake_zip_rasters(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + +/* Inflate names[i] out of `zip_path` to out_paths[i], streaming: a 32 KiB + * window, whatever the file weighs. This is the path for what the engine can + * only read BY PATH — an .mbtiles or .pmtiles it memory-maps — where the file + * must exist on disk but a second copy of it need not. A 4 GiB .mbtiles + * therefore costs 4 GiB, not 8 GiB. + * + * Serial by design: these are the big entries, the disk is the limit, and + * parallel writers only contend for it. `progress` fires after each file and + * may cancel; *out_done counts what was written. An entry the archive does not + * hold is skipped. */ +tile57_status tile57_zip_extract(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, + tile57_bake_progress progress, void *progress_ctx, + uint32_t *out_done, tile57_error *err); + /* Read a PMTiles archive's metadata JSON blob (decompressed) into *out / * *out_len (free with tile57_free); NULL/0 when the archive carries none. A * per-chart bake embeds the chart's M_COVR coverage + cscl + date/name under a diff --git a/src/bake_root.zig b/src/bake_root.zig index 01b19f7f..f6eae77b 100644 --- a/src/bake_root.zig +++ b/src/bake_root.zig @@ -29,6 +29,7 @@ pub const s101_adapter = root.s101_adapter; pub const catalogue = root.catalogue; pub const bake_enc = root.bake_enc; pub const auxfiles = root.auxfiles; +pub const zipsrc = root.zipsrc; // charts read straight out of a .zip pub const geometry = @import("geometry"); // integer geometry: boolean, plane, partition pub const portray = @import("portray"); diff --git a/src/capi.zig b/src/capi.zig index 0342425c..d3e615e9 100644 --- a/src/capi.zig +++ b/src/capi.zig @@ -17,6 +17,7 @@ const mariner = @import("style").mariner; const style = @import("style"); const errors = @import("errors"); // the engine error taxonomy + describe() const raster = @import("raster"); // raster charts (tile57_raster_chart_*) +const zipsrc = @import("zipsrc"); // charts read straight out of a .zip // The S-52 ColorProfiles/colorProfile.xml baked into the library (build.zig), so // the style C ABI generates colortables + a base style template with no on-disk // catalogue. Symbols/linestyles are NOT embedded here (only the bake exe needs them). @@ -412,7 +413,7 @@ export fn tile57_bake_files( in_list[i] = std.mem.span(ins[i]); out_list[i] = std.mem.span(outs[i]); } - const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label); + const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label, true); if (out_baked) |p| p.* = @intCast(baked); return OK; } @@ -422,9 +423,12 @@ const RasterJob = struct { next: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), done: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), cancel: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + /// Sheet paths, or — when `zip` is set — names of entries inside it. in: []const []const u8, out: []const []const u8, ok: []bool, + /// The archive the sheets are read out of, or null to read from disk. + zip: ?*const zipsrc.Archive = null, progress: chart.BakeProgress, label: chart.BakeLabel, ctx: ?*anyopaque, @@ -436,7 +440,14 @@ const RasterJob = struct { fn bakeOneRaster(io: std.Io, job: *RasterJob, i: usize) void { const a = std.heap.c_allocator; const stem = std.fs.path.stem(std.fs.path.basename(job.in[i])); - const kap = std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return; + // From the archive or from disk — the warp below cannot tell which. + const kap = blk: { + if (job.zip) |z| { + const idx = z.find(job.in[i]) orelse return; + break :blk z.readAlloc(a, io, idx, MAX_RASTER_BYTES) catch return; + } + break :blk std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return; + }; defer a.free(kap); const baked = raster.bakebsb.bakeBytes(a, kap, stem) catch return; defer a.free(baked.bytes); @@ -584,6 +595,174 @@ fn writeRasterPartition(out_list: []const []const u8, ok: []const bool) void { std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }) catch {}; } +// ---- charts inside a .zip ---------------------------------------------------- +// A chart archive is opened, read from, and closed within each call. Walking a +// 27,680-entry central directory costs about 8 ms, so the alternative — an open +// handle the host must hold, close, and keep off other threads — buys nothing +// and can be got wrong. + +/// The largest a single raster sheet may claim to expand to. A KAP runs to tens +/// of megabytes; the cap is here so a bad header fails instead of allocating. +const MAX_RASTER_BYTES: u64 = 512 << 20; + +/// List what a .zip holds: [{"name":..,"size":..,"packed":..}, ..] into +/// *out / *out_len (free with tile57_free). See tile57.h. +export fn tile57_zip_list(zip_path: ?[*:0]const u8, out: ?*?[*]u8, out_len: ?*usize, err: ?*CError) callconv(.c) c_int { + const o, const n = bytesOut(out, out_len) catch return failWith(err, .badarg, bad_out); + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + const json = arc.toJson(gpa) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(json); + // Copied with its terminator: the payload is length-delimited by *out_len + // AND readable as a C string, so a host can take either. + const p = exportAlloc(json.len + 1) orelse return failWith(err, .nomem, "out of memory"); + @memcpy(p[0 .. json.len + 1], json[0 .. json.len + 1]); + o.* = p; + n.* = json.len; + return OK; +} + +/// Inflate entries out of a .zip to paths the CALLER names, streaming. See tile57.h. +export fn tile57_zip_extract( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + progress: chart.BakeProgress, + progress_ctx: ?*anyopaque, + out_done: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_done) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + + // Serial on purpose: this is one file stream to disk per entry, and the + // entries that come this way are the big ones (a 4 GiB .mbtiles), where + // the disk is the limit and parallel writers only fight over it. + var done: u32 = 0; + for (0..n) |i| { + const name = std.mem.span(ns[i]); + const idx = arc.find(name) orelse continue; + arc.extractTo(io, idx, std.mem.span(outs[i])) catch continue; + done += 1; + if (progress) |cb| { + if (!cb(progress_ctx, @intCast(i + 1), @intCast(n))) break; + } + } + if (out_done) |p| p.* = done; + return OK; +} + +/// tile57_bake_files reading the cells STRAIGHT OUT of a .zip. See tile57.h. +export fn tile57_bake_zip_charts( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + for (0..n) |i| { + in_list[i] = std.mem.span(ns[i]); + out_list[i] = std.mem.span(outs[i]); + } + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + const baked = chart.bakeZipChartsToFiles(io, &arc, in_list, out_list, null, workers, progress, progress_ctx, label, true); + if (out_baked) |p| p.* = @intCast(baked); + return OK; +} + +/// tile57_bake_rasters reading the sheets STRAIGHT OUT of a .zip. See tile57.h. +export fn tile57_bake_zip_rasters( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + const ok = gpa.alloc(bool, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(ok); + @memset(ok, false); + for (0..n) |i| { + in_list[i] = std.mem.span(ns[i]); + out_list[i] = std.mem.span(outs[i]); + } + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + + var job = RasterJob{ + .in = in_list, + .out = out_list, + .ok = ok, + .zip = &arc, + .progress = progress, + .label = label, + .ctx = progress_ctx, + }; + // Same deep stacks as tile57_bake_rasters: the warp is the same work, + // only the bytes arrive from the archive instead of a file. + const stack = 16 * 1024 * 1024; + var threads: [8]std.Thread = undefined; + const want = @min(@max(workers, 1), @min(threads.len, n)); + var spawned: usize = 0; + while (spawned < want) : (spawned += 1) { + threads[spawned] = std.Thread.spawn(.{ .stack_size = stack }, rasterWorker, .{&job}) catch break; + } + if (spawned == 0) rasterWorker(&job); + for (threads[0..spawned]) |t| t.join(); + + var baked: u32 = 0; + for (ok) |o| { + if (o) baked += 1; + } + if (out_baked) |p| p.* = baked; + if (baked > 0) writeRasterPartition(out_list, ok); + return OK; +} + /// The metadata JSON blob of a PMTiles archive (decompressed) — e.g. the embedded /// per-cell "coverage" a single-cell bake carries — into *out / *out_len (free with /// tile57_free); NULL/0 when the archive carries none. See tile57.h. diff --git a/src/chart.zig b/src/chart.zig index 9c3378f3..07c747e5 100644 --- a/src/chart.zig +++ b/src/chart.zig @@ -35,6 +35,8 @@ const embedded_assets = @import("catalog"); // S-101 portrayal assets (renderVie const style = @import("style"); // displayDenomZ (the physical display-scale formula) const cell_coverage = @import("coverage"); // per-cell M_COVR coverage embedded in archive metadata const compose_mod = @import("compose"); // the runtime compositor (compose-backed view renders) +const zipsrc = @import("zipsrc"); // charts read straight out of a .zip +const auxfiles = @import("auxfiles"); // the text and pictures a cell points at // c_allocator, not smp_allocator: smp's per-CPU slab freelists never return // pages to the OS, so a long-lived host process's footprint ratchets up to the @@ -715,11 +717,13 @@ fn openCell(bytes: []const u8, rules_dir: ?[]const u8) ?*Chart { return src; } -/// A single ENC cell's on-disk bytes: base .000 + its sequential .001.. update chain. -const CellFiles = struct { +/// A single ENC cell's bytes: base .000 + its sequential .001.. update chain. +/// Where they came from is not recorded, so a cell read out of a directory and +/// one inflated out of a zip bake through the same path. +pub const CellFiles = struct { base: []u8, updates: [][]u8, - fn deinit(self: *CellFiles) void { + pub fn deinit(self: *CellFiles) void { gpa.free(self.base); for (self.updates) |u| gpa.free(u); gpa.free(self.updates); @@ -763,6 +767,99 @@ fn readCellFiles(path: []const u8) !CellFiles { return .{ .base = base, .updates = try updates.toOwnedSlice(gpa) }; } +/// The most a single ENC cell may claim to expand to. Cells run to a few MiB; +/// a header promising more than this is a damaged or hostile archive, and the +/// point of a cap is to find that out before allocating rather than after. +pub const MAX_CELL_BYTES: u64 = 256 << 20; + +/// `readCellFiles` out of a zip: the base entry plus its update chain, each +/// inflated on its own. Nothing is written to disk and nothing larger than one +/// cell is held. +pub fn readCellFromZip(io: std.Io, arc: *const zipsrc.Archive, idx: usize) !CellFiles { + const base = try arc.readAlloc(gpa, io, idx, MAX_CELL_BYTES); + errdefer gpa.free(base); + + const up_idx = try arc.updatesFor(gpa, idx); + defer gpa.free(up_idx); + var updates = std.ArrayList([]u8).empty; + errdefer { + for (updates.items) |u| gpa.free(u); + updates.deinit(gpa); + } + // A broken update stops the chain rather than the cell: an ENC applied + // through update 3 is a chart, and refusing to draw it because update 4 + // is corrupt leaves the mariner with nothing. + for (up_idx) |ui| { + const ub = arc.readAlloc(gpa, io, ui, MAX_CELL_BYTES) catch break; + updates.append(gpa, ub) catch { + gpa.free(ub); + break; + }; + } + return .{ .base = base, .updates = try updates.toOwnedSlice(gpa) }; +} + +/// The directory to write a chart's referenced files into, or null when the +/// caller did not give this chart a directory of its own. +/// +/// A cell's .TXT and pictures are named per exchange set, not per chart — +/// US1EEZ3M references US1EEZ3A.TXT — so several charts baked flat into one +/// directory would share a manifest and overwrite each other's. Rather than +/// guess, the rule is explicit: aux files are written only when the archive +/// sits in a directory named for the chart (/US1EEZ3M/US1EEZ3M.pmtiles), +/// which is the exchange set's own shape and what tile57_aux_open expects. +fn auxDirFor(out_path: []const u8, stem: []const u8) ?[]const u8 { + const dir = std.fs.path.dirname(out_path) orelse return null; + if (!std.mem.eql(u8, std.fs.path.basename(dir), stem)) return null; + return dir; +} + +/// Write the text and pictures a cell references beside its baked archive, out +/// of the cell's own directory in the archive. Best-effort: a chart still +/// draws without its caution notes, so a failure here is not a bake failure. +fn writeAuxFromZip(io: std.Io, arc: *const zipsrc.Archive, idx: usize, out_path: []const u8) void { + const stem = std.fs.path.stem(std.fs.path.basename(arc.entries[idx].name)); + const dst = auxDirFor(out_path, stem) orelse return; + + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + var files = std.ArrayList(auxfiles.File).empty; + for (arc.siblingsOf(idx)) |si| { + const name = arc.entries[si].name; + if (!auxfiles.isContent(name)) continue; + const bytes = arc.readAlloc(a, io, si, MAX_CELL_BYTES) catch continue; + files.append(a, .{ .owner = stem, .name = name, .bytes = bytes }) catch continue; + } + _ = auxfiles.writeDir(io, a, dst, files.items) catch {}; +} + +/// The same, for a cell read from a directory: its referenced files are the +/// aux content sitting beside it. +fn writeAuxFromDir(io: std.Io, cell_path: []const u8, out_path: []const u8) void { + const stem = std.fs.path.stem(std.fs.path.basename(cell_path)); + const dst = auxDirFor(out_path, stem) orelse return; + + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + const src_dir = std.fs.path.dirname(cell_path) orelse "."; + var dir = std.Io.Dir.cwd().openDir(io, src_dir, .{ .iterate = true }) catch return; + defer dir.close(io); + + var files = std.ArrayList(auxfiles.File).empty; + var it = dir.iterate(); + while (it.next(io) catch null) |ent| { + if (ent.kind != .file or !auxfiles.isContent(ent.name)) continue; + // The iterator reuses its name buffer, so both the name and the bytes + // must be copied before the next step. + const name = a.dupe(u8, ent.name) catch continue; + const bytes = dir.readFileAlloc(io, name, a, .unlimited) catch continue; + files.append(a, .{ .owner = stem, .name = name, .bytes = bytes }) catch continue; + } + _ = auxfiles.writeDir(io, a, dst, files.items) catch {}; +} + /// Bake a SINGLE .000 cell (+ updates) to a PMTiles archive over its NATIVE band's /// zoom range (`bandZooms(bandOf(cscl))`) and nothing else — the composite model bakes /// each cell at its own compilation scale; the stitcher combines them and handles any @@ -773,13 +870,22 @@ fn readCellFiles(path: []const u8) !CellFiles { /// so the composite stitcher rebuilds the ownership partition from the baked archives /// without re-parsing the .000. Read it back with `decodedCoverageFromArchive`. pub fn bakeChartBytes(cell_path: []const u8, rules_dir: ?[]const u8) !?[]u8 { + var cf = try readCellFiles(cell_path); + defer cf.deinit(); + return bakeCellFiles(&cf, cell_path, rules_dir); +} + +/// `bakeChartBytes` for a cell ALREADY IN MEMORY. `cell_name` is the cell's +/// name — a path or a zip entry name; only its stem is read, as the ownership +/// tie-break and the pick report's source-cell badge. Nothing is read from +/// disk, so this is the entry point for a cell inflated straight out of an +/// archive. +pub fn bakeCellFiles(cf: *const CellFiles, cell_name: []const u8, rules_dir: ?[]const u8) !?[]u8 { // Populate the read-only portrayal globals (feature catalogue + complex-linestyle table) // before portraying: without them, complex lines fall back to plain geometry and their S-52 // linestyle is dropped from the tile. Idempotent; in the parallel batch path bakeChartsParallel // has already warmed up before spawning workers, so this is a no-op there (and race-free). warmup(); - var cf = try readCellFiles(cell_path); - defer cf.deinit(); // Capture coverage for the embedded sidecar (one cheap parse). The stem is the // ownership tie-break name — matches the coverage loader. @@ -792,7 +898,7 @@ pub fn bakeChartBytes(cell_path: []const u8, rules_dir: ?[]const u8) !?[]u8 { // The dataset stem is the ownership tie-break name AND the pick-report's // "source cell" badge — pass it into the tile bake below (bakeArchive borrows // it for cell.name), or every feature's `cell` prop bakes empty. - const stem = std.fs.path.stem(std.fs.path.basename(cell_path)); + const stem = std.fs.path.stem(std.fs.path.basename(cell_name)); var cscl: i32 = s57.peekScale(gpa, cf.base) orelse 0; if (parseAnyCell(cf.base, cf.updates)) |loaded| { var cell = loaded.cell; @@ -888,9 +994,14 @@ pub const BakeLabel = ?*const fn (?*anyopaque, u32) callconv(.c) void; const BakeFileCtx = struct { next: std.atomic.Value(usize), + /// Cell paths, or — when `zip` is set — names of entries inside it. in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, + /// The archive the cells are read out of, or null to read from disk. The + /// archive is immutable once opened and every read opens its own handle, + /// so all the workers share this one. + zip: ?*const zipsrc.Archive = null, io: std.Io, ok: []bool, ms: []i64, // per-cell wall time — the bake profiles itself (slowest cells printed at the end) @@ -900,6 +1011,8 @@ const BakeFileCtx = struct { done: std.atomic.Value(u32), /// Set when a progress callback returned false; every worker drains out at its next cell. cancel: std.atomic.Value(bool), + /// Write the text and pictures each cell references beside its archive. + aux: bool = true, }; fn bakeOneToFile(ctx: *BakeFileCtx, i: usize) void { @@ -908,9 +1021,30 @@ fn bakeOneToFile(ctx: *BakeFileCtx, i: usize) void { const t1 = std.Io.Clock.awake.now(ctx.io); ctx.ms[i] = @intCast(@divTrunc(t1.nanoseconds - t0.nanoseconds, 1_000_000)); } - const arc = (bakeChartBytes(ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + // From the archive or from disk — the bake below cannot tell which. + var zip_idx: ?usize = null; + const arc = blk: { + if (ctx.zip) |z| { + const idx = z.find(ctx.in_paths[i]) orelse return; + zip_idx = idx; + var cf = readCellFromZip(ctx.io, z, idx) catch return; + defer cf.deinit(); + break :blk (bakeCellFiles(&cf, ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + } + break :blk (bakeChartBytes(ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + }; defer freeBytes(arc); std.Io.Dir.cwd().writeFile(ctx.io, .{ .sub_path = ctx.out_paths[i], .data = arc }) catch return; + // The files this cell references, beside it — a pick report resolves + // TXTDSC by name, so a chart without them answers "see US1EEZ3A.TXT" and + // cannot show it. + if (ctx.aux) { + if (zip_idx) |idx| { + writeAuxFromZip(ctx.io, ctx.zip.?, idx, ctx.out_paths[i]); + } else { + writeAuxFromDir(ctx.io, ctx.in_paths[i], ctx.out_paths[i]); + } + } var digest: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(arc, &digest, .{}); const hex = std.fmt.bytesToHex(digest, .lower); @@ -945,7 +1079,26 @@ fn bakeFileWorker(ctx: *BakeFileCtx) void { /// after each cell and may CANCEL by returning false (see BakeProgress); `label(progress_ctx, i)` /// fires beside it and names the chart that finished. Race-free (warms up first; each bake is /// independent). Returns the count written — fewer than in_paths.len when cancelled. -pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel) usize { +/// +/// `aux` writes the text and pictures each cell references beside its archive +/// (see auxDirFor for when that is possible). +pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { + return bakeToFiles(io, null, in_paths, out_paths, rules_dir, workers, progress, progress_ctx, label, aux); +} + +/// `bakeChartsToFiles` reading every cell STRAIGHT OUT of `arc`: `names` are +/// entry names inside the archive rather than paths, and nothing is unzipped +/// — each cell and its updates are inflated when their turn comes and freed +/// with the archive they bake into. Workers pull from one open archive +/// concurrently, so this runs at the same rate as the on-disk bake. +/// +/// A name the archive does not hold is skipped, like a cell that fails to +/// bake: *out_baked counts what was written. +pub fn bakeZipChartsToFiles(io: std.Io, arc: *const zipsrc.Archive, names: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { + return bakeToFiles(io, arc, names, out_paths, rules_dir, workers, progress, progress_ctx, label, aux); +} + +fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { std.debug.assert(out_paths.len == in_paths.len); if (in_paths.len == 0) return 0; warmup(); @@ -955,7 +1108,7 @@ pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: [] const cell_ms = gpa.alloc(i64, in_paths.len) catch return 0; defer gpa.free(cell_ms); @memset(cell_ms, 0); - var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false) }; + var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux }; var n = @min(@max(workers, 1), in_paths.len); if (n > MAX_BAKE_WORKERS) n = MAX_BAKE_WORKERS; if (n <= 1) { @@ -1038,7 +1191,7 @@ pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: out_paths.append(a, out_path) catch continue; } if (in_paths.items.len == 0) return 0; - return bakeChartsToFiles(io, in_paths.items, out_paths.items, rules_dir, workers, progress, progress_ctx, label); + return bakeChartsToFiles(io, in_paths.items, out_paths.items, rules_dir, workers, progress, progress_ctx, label, true); } /// The file's modification time in nanoseconds, or null if it doesn't exist / can't be statted. diff --git a/src/root.zig b/src/root.zig index ad049904..5012c3a6 100644 --- a/src/root.zig +++ b/src/root.zig @@ -23,7 +23,8 @@ pub const catalogue = s101.catalogue; pub const bake_enc = @import("scene").bake_enc; // banded multi-cell ENC_ROOT -> PMTiles pub const style = @import("style"); // colortables, line styles, and style.json generation pub const mariner = @import("style").mariner; // mariner-driven MapLibre style patching -pub const auxfiles = @import("auxfiles.zig"); // the text and pictures a cell points at +pub const auxfiles = @import("auxfiles"); // the text and pictures a cell points at +pub const zipsrc = @import("zipsrc"); // charts read straight out of a .zip // capi (the C ABI) lives in lib_root.zig so the test/bake exes stay pure Zig. test { diff --git a/src/zipsrc.zig b/src/zipsrc.zig new file mode 100644 index 00000000..566142b3 --- /dev/null +++ b/src/zipsrc.zig @@ -0,0 +1,578 @@ +//! Charts read STRAIGHT OUT of a .zip, one entry at a time. +//! +//! A chart archive is big: NOAA's All_ENCs.zip is 788 MB of deflate holding +//! 2.0 GiB across 27,680 entries — 7,224 cells, their 4,414 updates, and +//! 15,539 .TXT the pick report resolves by name. Unzipping it first costs the +//! mariner 2.0 GiB of disk that is dead the moment the bake ends, on top of +//! the baked charts they actually keep. So nothing here unzips: the central +//! directory is walked once (2 ms for those 27,680 entries), and each file is +//! inflated on its own, when it is needed, into a buffer its own size. +//! +//! Two properties make that work, and both are load-bearing: +//! +//! * Entries are randomly addressable — seek to the local header, inflate. +//! A read does not disturb any other read, so `readAlloc` opens its own +//! handle and the parallel bake keeps every core busy pulling different +//! cells out of one archive. There is no shared cursor to lock. +//! * Inflating is bounded — a 32 KiB window plus the read buffer, whatever +//! the entry's size. `extractTo` therefore streams a 4 GiB .mbtiles to +//! disk in constant memory, and needs no space for a second copy. +//! +//! Nothing here decides WHAT a chart is. The archive reports its names and +//! reads whichever ones it is asked for; the host classifies. What it does +//! know is the exchange set's SHAPE, because that is structure rather than +//! policy: a cell's .001.. updates belong to that cell (`updatesFor`), and +//! the files sitting in a cell's own directory are the ones it references +//! (`siblingsOf`) — the .TXT a TXTDSC names, the picture a PICREP names. + +const std = @import("std"); +const zip = std.zip; +const flate = std.compress.flate; +const Allocator = std.mem.Allocator; + +pub const Error = error{ + BadLocalHeader, + UnsupportedCompressionMethod, + EntryTooLarge, +}; + +pub const Entry = struct { + /// The name as stored, e.g. "ENC_ROOT/US5MD12M/US5MD12M.000". Arena-owned. + name: []const u8, + uncompressed_size: u64, + compressed_size: u64, + raw: zip.Iterator.Entry, +}; + +/// A zip opened for reading charts. Read-only once opened, so any number of +/// threads may `readAlloc`/`extractTo` from one `Archive` at the same time. +pub const Archive = struct { + gpa: Allocator, + arena: std.heap.ArenaAllocator, + /// Our own copy of the archive's path: each read opens the file again + /// rather than sharing a cursor. + path: []const u8, + entries: []Entry, + by_name: std.StringHashMapUnmanaged(usize), + /// Directory prefix (no trailing '/') -> the entries directly in it. Built + /// once, because the alternative is rescanning 27,680 names per cell. + by_dir: std.StringHashMapUnmanaged([]usize), + + /// Walk the central directory. Directory entries (trailing '/') are left + /// out — they are never charts. Encrypted and multi-disk archives are + /// rejected here, by `zip.Iterator`, rather than at the first read. + pub fn open(gpa: Allocator, io: std.Io, path: []const u8) !Archive { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const a = arena.allocator(); + + const f = try std.Io.Dir.cwd().openFile(io, path, .{}); + defer f.close(io); + var rbuf: [64 * 1024]u8 = undefined; + var fr = f.reader(io, &rbuf); + + var it = try zip.Iterator.init(&fr); + var entries: std.ArrayList(Entry) = .empty; + errdefer entries.deinit(gpa); + try entries.ensureTotalCapacity(gpa, @intCast(@min(it.cd_record_count, 1 << 20))); + + var name_buf: [4096]u8 = undefined; + while (try it.next()) |e| { + if (e.filename_len == 0 or e.filename_len > name_buf.len) continue; + const name = name_buf[0..e.filename_len]; + try fr.seekTo(e.header_zip_offset + @sizeOf(zip.CentralDirectoryFileHeader)); + try fr.interface.readSliceAll(name); + if (name[name.len - 1] == '/') continue; // a directory, not a chart + try entries.append(gpa, .{ + .name = try a.dupe(u8, name), + .uncompressed_size = e.uncompressed_size, + .compressed_size = e.compressed_size, + .raw = e, + }); + } + + const owned = try entries.toOwnedSlice(gpa); + errdefer gpa.free(owned); + var by_name: std.StringHashMapUnmanaged(usize) = .empty; + errdefer by_name.deinit(gpa); + try by_name.ensureTotalCapacity(gpa, @intCast(owned.len)); + for (owned, 0..) |e, i| by_name.putAssumeCapacity(e.name, i); + + // Group by directory in one pass, into the arena, so `siblingsOf` is a + // lookup rather than a scan. + var grow: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(usize)) = .empty; + defer { + var it2 = grow.valueIterator(); + while (it2.next()) |v| v.deinit(gpa); + grow.deinit(gpa); + } + for (owned, 0..) |e, i| { + const d = dirNameOf(e.name); + const slot = try grow.getOrPut(gpa, d); + if (!slot.found_existing) slot.value_ptr.* = .empty; + try slot.value_ptr.append(gpa, i); + } + var by_dir: std.StringHashMapUnmanaged([]usize) = .empty; + errdefer by_dir.deinit(gpa); + try by_dir.ensureTotalCapacity(gpa, grow.count()); + var git = grow.iterator(); + while (git.next()) |kv| { + by_dir.putAssumeCapacity(kv.key_ptr.*, try a.dupe(usize, kv.value_ptr.items)); + } + + // Every arena allocation must happen BEFORE the literal below copies + // the arena: the copy freezes the buffer list, so anything allocated + // while building the literal lands in a node the copy cannot free. + const path_copy = try a.dupe(u8, path); + return .{ + .gpa = gpa, + .arena = arena, + .path = path_copy, + .entries = owned, + .by_name = by_name, + .by_dir = by_dir, + }; + } + + pub fn deinit(self: *Archive) void { + self.by_name.deinit(self.gpa); + self.by_dir.deinit(self.gpa); + self.gpa.free(self.entries); + self.arena.deinit(); + } + + pub fn find(self: *const Archive, name: []const u8) ?usize { + return self.by_name.get(name); + } + + /// Inflate entry `i` into a fresh buffer (caller frees). `max` bounds what + /// the archive is allowed to claim an entry expands to — an ENC cell is a + /// few MiB, so a header promising gigabytes is a reason to stop, not to + /// allocate. Opens its own file handle: safe to call from many threads. + pub fn readAlloc(self: *const Archive, gpa: Allocator, io: std.Io, i: usize, max: u64) ![]u8 { + const e = self.entries[i]; + if (e.uncompressed_size > max) return Error.EntryTooLarge; + const buf = try gpa.alloc(u8, @intCast(e.uncompressed_size)); + errdefer gpa.free(buf); + + var w: std.Io.Writer = .fixed(buf); + try self.streamEntry(io, i, &w); + if (w.end != buf.len) return error.EndOfStream; + return buf; + } + + /// Inflate entry `i` straight to `out_path`, in constant memory however + /// large it is — the one way a 4 GiB .mbtiles lands on disk without a + /// second copy of it existing anywhere. `out_path` is the CALLER's name + /// for the file, never the archive's, so a hostile entry name cannot + /// choose where this writes. + pub fn extractTo(self: *const Archive, io: std.Io, i: usize, out_path: []const u8) !void { + const out = try std.Io.Dir.cwd().createFile(io, out_path, .{ .truncate = true }); + defer out.close(io); + var wbuf: [256 * 1024]u8 = undefined; + var fw = out.writer(io, &wbuf); + try self.streamEntry(io, i, &fw.interface); + try fw.interface.flush(); + } + + /// The `.001..` update chain belonging to the `.000` cell at `i`, in order, + /// stopping at the first gap — the same rule the on-disk reader applies to + /// a cell's directory, applied to the archive's names instead. Empty when + /// `i` is not a base cell. Caller frees. + pub fn updatesFor(self: *const Archive, gpa: Allocator, i: usize) ![]usize { + var out: std.ArrayList(usize) = .empty; + errdefer out.deinit(gpa); + const name = self.entries[i].name; + if (name.len < 4 or !std.mem.eql(u8, name[name.len - 4 ..], ".000")) { + return out.toOwnedSlice(gpa); + } + const stem = name[0 .. name.len - 4]; + var buf: [4096]u8 = undefined; + var u: u32 = 1; + while (u <= 999) : (u += 1) { + const up = std.fmt.bufPrint(&buf, "{s}.{d:0>3}", .{ stem, u }) catch break; + const idx = self.find(up) orelse break; + try out.append(gpa, idx); + } + return out.toOwnedSlice(gpa); + } + + /// Every entry in the same directory as `i`, `i` included. The exchange + /// set puts a cell's referenced text and pictures in the cell's own + /// directory, so this is the candidate list an aux-file pass filters — + /// the archive does not decide which of them are content, which is why it + /// does not drop the cell either. Borrowed; valid until deinit. + pub fn siblingsOf(self: *const Archive, i: usize) []const usize { + return self.by_dir.get(dirNameOf(self.entries[i].name)) orelse &.{}; + } + + /// Every entry as JSON: [{"name":..,"size":..,"packed":..}, ..]. The host + /// classifies from this; the archive takes no view on which are charts. + /// NUL-terminated so it can cross the C ABI as a plain string. + pub fn toJson(self: *const Archive, gpa: Allocator) ![:0]u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(gpa); + try buf.append(gpa, '['); + for (self.entries, 0..) |e, i| { + if (i != 0) try buf.append(gpa, ','); + try buf.appendSlice(gpa, "{\"name\":"); + try appendJsonString(gpa, &buf, e.name); + try buf.print(gpa, ",\"size\":{d},\"packed\":{d}}}", .{ e.uncompressed_size, e.compressed_size }); + } + try buf.append(gpa, ']'); + return buf.toOwnedSliceSentinel(gpa, 0); + } + + /// Where entry `i`'s compressed bytes start: past its LOCAL header, whose + /// name and extra fields may be sized differently from the central + /// directory's copy. + fn dataOffset(e: zip.Iterator.Entry, fr: *std.Io.File.Reader) !u64 { + try fr.seekTo(e.file_offset); + const lh = try fr.interface.takeStruct(zip.LocalFileHeader, .little); + if (!std.mem.eql(u8, &lh.signature, &zip.local_file_header_sig)) return Error.BadLocalHeader; + return e.file_offset + @sizeOf(zip.LocalFileHeader) + lh.filename_len + lh.extra_len; + } + + /// Inflate one entry into `w`. Bounded: a 32 KiB window and the read + /// buffer, whatever the entry weighs. + fn streamEntry(self: *const Archive, io: std.Io, i: usize, w: *std.Io.Writer) !void { + const e = self.entries[i].raw; + const f = try std.Io.Dir.cwd().openFile(io, self.path, .{}); + defer f.close(io); + var rbuf: [64 * 1024]u8 = undefined; + var fr = f.reader(io, &rbuf); + try fr.seekTo(try dataOffset(e, &fr)); + switch (e.compression_method) { + .store => try fr.interface.streamExact64(w, e.uncompressed_size), + .deflate => { + var window: [flate.max_window_len]u8 = undefined; + var d: flate.Decompress = .init(&fr.interface, .raw, &window); + try d.reader.streamExact64(w, e.uncompressed_size); + }, + else => return Error.UnsupportedCompressionMethod, + } + } +}; + +/// The directory part of a zip entry name ("" at the top level). Zip names are +/// always '/'-separated, whatever the platform, so this is not path logic. +fn dirNameOf(name: []const u8) []const u8 { + const slash = std.mem.lastIndexOfScalar(u8, name, '/') orelse return ""; + return name[0..slash]; +} + +fn appendJsonString(gpa: Allocator, buf: *std.ArrayList(u8), s: []const u8) !void { + try buf.append(gpa, '"'); + for (s) |ch| switch (ch) { + '"' => try buf.appendSlice(gpa, "\\\""), + '\\' => try buf.appendSlice(gpa, "\\\\"), + '\n' => try buf.appendSlice(gpa, "\\n"), + '\r' => try buf.appendSlice(gpa, "\\r"), + '\t' => try buf.appendSlice(gpa, "\\t"), + else => if (ch < 0x20) { + try buf.print(gpa, "\\u{x:0>4}", .{ch}); + } else try buf.append(gpa, ch), + }; + try buf.append(gpa, '"'); +} + +// --------------------------------------------------------------------------- +// Tests. The fixtures are real zips, written here rather than committed: the +// reader's whole job is tolerating what a writer actually produces (a local +// header sized differently from the central one, a stored entry beside a +// deflated one), and a committed blob would freeze one writer's habits. +// --------------------------------------------------------------------------- + +const testing = std.testing; + +const TestFile = struct { name: []const u8, data: []const u8, deflate: bool }; + +/// Write a minimal but valid zip: local header + data per file, then the +/// central directory, then the end record. +fn writeTestZip(gpa: Allocator, io: std.Io, path: []const u8, files: []const TestFile) !void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + + var offsets = try gpa.alloc(u32, files.len); + defer gpa.free(offsets); + var packed_lens = try gpa.alloc(u32, files.len); + defer gpa.free(packed_lens); + var crcs = try gpa.alloc(u32, files.len); + defer gpa.free(crcs); + + for (files, 0..) |f, i| { + offsets[i] = @intCast(out.items.len); + crcs[i] = std.hash.Crc32.hash(f.data); + + var body: std.Io.Writer.Allocating = try .initCapacity(gpa, 4096); + defer body.deinit(); + if (f.deflate) { + var window: [flate.max_window_len]u8 = undefined; + var c = try flate.Compress.init(&body.writer, &window, .raw, .default); + try c.writer.writeAll(f.data); + try c.finish(); + } else { + try body.writer.writeAll(f.data); + } + const body_bytes = body.written(); + packed_lens[i] = @intCast(body_bytes.len); + + const lh: zip.LocalFileHeader = .{ + .signature = zip.local_file_header_sig, + .version_needed_to_extract = 20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = if (f.deflate) .deflate else .store, + .last_modification_time = 0, + .last_modification_date = 0, + .crc32 = crcs[i], + .compressed_size = packed_lens[i], + .uncompressed_size = @intCast(f.data.len), + .filename_len = @intCast(f.name.len), + .extra_len = 0, + }; + try out.appendSlice(gpa, std.mem.asBytes(&lh)); + try out.appendSlice(gpa, f.name); + try out.appendSlice(gpa, body_bytes); + } + + const cd_start: u32 = @intCast(out.items.len); + for (files, 0..) |f, i| { + const ch: zip.CentralDirectoryFileHeader = .{ + .signature = zip.central_file_header_sig, + .version_made_by = 20, + .version_needed_to_extract = 20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = if (f.deflate) .deflate else .store, + .last_modification_time = 0, + .last_modification_date = 0, + .crc32 = crcs[i], + .compressed_size = packed_lens[i], + .uncompressed_size = @intCast(f.data.len), + .filename_len = @intCast(f.name.len), + .extra_len = 0, + .comment_len = 0, + .disk_number = 0, + .internal_file_attributes = 0, + .external_file_attributes = 0, + .local_file_header_offset = offsets[i], + }; + try out.appendSlice(gpa, std.mem.asBytes(&ch)); + try out.appendSlice(gpa, f.name); + } + const cd_size: u32 = @intCast(out.items.len - cd_start); + + const end: zip.EndRecord = .{ + .signature = zip.end_record_sig, + .disk_number = 0, + .central_directory_disk_number = 0, + .record_count_disk = @intCast(files.len), + .record_count_total = @intCast(files.len), + .central_directory_size = cd_size, + .central_directory_offset = cd_start, + .comment_len = 0, + }; + try out.appendSlice(gpa, std.mem.asBytes(&end)); + + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = out.items }); +} + +/// tmpDir lives under the build root's .zig-cache; the reader takes paths +/// through cwd(), so the tests need its path rather than its handle. +fn tmpPath(gpa: Allocator, tmp: *std.testing.TmpDir) ![]u8 { + return std.fs.path.join(gpa, &.{ ".zig-cache", "tmp", &tmp.sub_path }); +} + +fn testIo() std.Io { + return std.Io.Threaded.global_single_threaded.io(); +} + +test "reads a stored and a deflated entry byte for byte" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // Long and repetitive, so deflate actually has something to do. + const big = "US5MD12M soundings " ** 400; + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/", .data = "", .deflate = false }, + .{ .name = "ENC_ROOT/A/A.000", .data = "stored cell bytes", .deflate = false }, + .{ .name = "ENC_ROOT/B/B.000", .data = big, .deflate = true }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The directory entry is not a chart and is not listed. + try testing.expectEqual(@as(usize, 2), arc.entries.len); + try testing.expectEqualStrings("ENC_ROOT/A/A.000", arc.entries[0].name); + + const a = try arc.readAlloc(gpa, io, 0, 1 << 20); + defer gpa.free(a); + try testing.expectEqualStrings("stored cell bytes", a); + + const b = try arc.readAlloc(gpa, io, arc.find("ENC_ROOT/B/B.000").?, 1 << 20); + defer gpa.free(b); + try testing.expectEqualStrings(big, b); + // Deflate really was exercised, not silently stored. + try testing.expect(arc.entries[1].compressed_size < arc.entries[1].uncompressed_size); +} + +test "an entry that claims more than the cap is refused before allocating" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "big.000", .data = "0123456789", .deflate = false }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + try testing.expectError(Error.EntryTooLarge, arc.readAlloc(gpa, io, 0, 4)); +} + +test "the update chain follows the cell and stops at the first gap" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // .003 is missing, so .004 is not part of the chain even though it is in + // the archive. A cell in another directory with the same stem must not be + // mistaken for an update of this one. + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = "base", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.001", .data = "u1", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.002", .data = "u2", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.004", .data = "u4", .deflate = false }, + .{ .name = "OTHER/US5MD12M/US5MD12M.003", .data = "elsewhere", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.TXT", .data = "notes", .deflate = false }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + const base = arc.find("ENC_ROOT/US5MD12M/US5MD12M.000").?; + const ups = try arc.updatesFor(gpa, base); + defer gpa.free(ups); + try testing.expectEqual(@as(usize, 2), ups.len); + try testing.expectEqualStrings("ENC_ROOT/US5MD12M/US5MD12M.001", arc.entries[ups[0]].name); + try testing.expectEqualStrings("ENC_ROOT/US5MD12M/US5MD12M.002", arc.entries[ups[1]].name); + + // A .TXT is not a base cell, so it has no chain. + const txt = arc.find("ENC_ROOT/US5MD12M/US5MD12M.TXT").?; + const none = try arc.updatesFor(gpa, txt); + defer gpa.free(none); + try testing.expectEqual(@as(usize, 0), none.len); +} + +test "a cell's directory names the files it references" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = "base", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US348MDE.TXT", .data = "caution", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/PIC1.TIF", .data = "pic", .deflate = false }, + .{ .name = "ENC_ROOT/US4MD11M/US4MD11M.000", .data = "other", .deflate = false }, + .{ .name = "ENC_ROOT/US4MD11M/OTHER.TXT", .data = "not mine", .deflate = false }, + .{ .name = "CATALOG.031", .data = "catalogue", .deflate = false }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The cell's own directory, and nothing from the cell next door. + const base = arc.find("ENC_ROOT/US5MD12M/US5MD12M.000").?; + const sibs = arc.siblingsOf(base); + try testing.expectEqual(@as(usize, 3), sibs.len); + for (sibs) |si| { + try testing.expect(std.mem.startsWith(u8, arc.entries[si].name, "ENC_ROOT/US5MD12M/")); + } + + // An entry at the top level has no directory to share. + const cat = arc.find("CATALOG.031").?; + try testing.expectEqual(@as(usize, 1), arc.siblingsOf(cat).len); +} + +test "extractTo writes the entry under the caller's name" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + const body = "tiles and more tiles " ** 500; + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "../evil/USA.mbtiles", .data = body, .deflate = true }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The archive's own name is ignored: the destination is ours. + const out = try std.fs.path.join(gpa, &.{ dir, "safe.mbtiles" }); + defer gpa.free(out); + try arc.extractTo(io, 0, out); + + const got = try std.Io.Dir.cwd().readFileAlloc(io, out, gpa, .unlimited); + defer gpa.free(got); + try testing.expectEqualStrings(body, got); +} + +test "the listing is JSON and NUL-terminated" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "a\"quote\".000", .data = "xy", .deflate = false }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + const json = try arc.toJson(gpa); + defer gpa.free(json); + // The host reads this as a C string; without the terminator it reads on. + try testing.expectEqual(@as(u8, 0), json[json.len]); + try testing.expectEqualStrings("[{\"name\":\"a\\\"quote\\\".000\",\"size\":2,\"packed\":2}]", json); +} + +test "a file that is not a zip is refused at open" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "not.zip" }); + defer gpa.free(zpath); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = "0001 this is an S-57 cell, not an archive" }); + try testing.expectError(error.ZipNoEndRecord, Archive.open(gpa, io, zpath)); +} diff --git a/tools/bake.zig b/tools/bake.zig index bc6472b0..fa60513e 100644 --- a/tools/bake.zig +++ b/tools/bake.zig @@ -1,4 +1,4 @@ -//! `bake -o [--rules DIR] +//! `bake -o [--rules DIR] //! [-j N]` — produce a LIVE-composite structure on disk. Bake each chart to its OWN native-scale PMTiles under //! `//` (with its M_COVR coverage embedded in the metadata), then open a resident //! compositor over them and write the ownership partition to `/partition.tpart`. There is @@ -13,6 +13,7 @@ const compose = @import("compose"); // openComposeSourceFiles + serializePartiti const raster = @import("raster"); // the BSB/KAP bake + the raster chart handle const common = @import("common.zig"); const auxfiles = @import("engine").auxfiles; +const zipsrc = @import("engine").zipsrc; const Flags = common.Flags; const usageErr = common.usageErr; const resolveRulesDir = common.resolveRulesDir; @@ -168,34 +169,31 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { // The per-chart archive paths that back the compositor. var archive_paths = std.ArrayList([]const u8).empty; - var aux_written: usize = 0; { - // Bake each chart (dedup by stem — a boundary chart shared by two districts bakes once) - // to its own //.pmtiles, the shape an exchange set uses. + // Bake each chart to its own directory, the shape an exchange set uses. + // From a tree: dedup by stem (a boundary chart shared by two districts + // bakes once) into //.pmtiles. From an archive: + // mirror the entry's path, so what comes out is laid out like what + // went in. try std.Io.Dir.cwd().createDirPath(io, out_dir); var cell_paths = std.ArrayList([]const u8).empty; - var aux_files = std.ArrayList(auxfiles.File).empty; - if (std.mem.endsWith(u8, base_path, ".000")) { - try cell_paths.append(a, base_path); - // The cell's own directory holds the files it references. - if (want_aux) { - const stem = std.fs.path.stem(std.fs.path.basename(base_path)); - if (std.fs.path.dirname(base_path)) |cell_dir| { - var dir = std.Io.Dir.cwd().openDir(io, cell_dir, .{ .iterate = true }) catch null; - if (dir) |*d| { - defer d.close(io); - var it = d.iterate(); - while (it.next(io) catch null) |entry| { - if (entry.kind != .file or !auxfiles.isContent(entry.name)) continue; - const p = std.fs.path.join(a, &.{ cell_dir, entry.name }) catch continue; - const bytes = std.Io.Dir.cwd().readFileAlloc(io, p, a, .unlimited) catch continue; - aux_files.append(a, .{ .owner = stem, .name = entry.name, .bytes = bytes }) catch {}; - } - } - } + // A .zip is baked where it lies: the cells are named, not extracted, + // and each one is inflated as its turn comes. Nothing is unzipped, so + // importing NOAA's 788 MB All_ENCs.zip does not first cost 2 GiB of + // disk for source data that is dead as soon as the bake ends. + var zip_arc: ?*zipsrc.Archive = null; + if (std.mem.endsWith(u8, base_path, ".zip")) { + const arc = a.create(zipsrc.Archive) catch return usageErr("out of memory"); + arc.* = zipsrc.Archive.open(a, io, base_path) catch return usageErr("cannot read archive"); + zip_arc = arc; + for (arc.entries) |e| { + if (std.mem.endsWith(u8, e.name, ".000")) cell_paths.append(a, e.name) catch {}; } + std.debug.print("{s}: {d} entries, {d} cell(s)\n", .{ base_path, arc.entries.len, cell_paths.items.len }); + } else if (std.mem.endsWith(u8, base_path, ".000")) { + try cell_paths.append(a, base_path); } else { var dir = std.Io.Dir.cwd().openDir(io, base_path, .{ .iterate = true }) catch return usageErr("cannot open ENC_ROOT"); defer dir.close(io); @@ -207,16 +205,9 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { while (walker.next(io) catch null) |entry| { scan.step(); if (entry.kind != .file) continue; - if (want_aux and auxfiles.isContent(entry.path)) { - const p = std.fs.path.join(a, &.{ base_path, entry.path }) catch continue; - const bytes = std.Io.Dir.cwd().readFileAlloc(io, p, a, .unlimited) catch continue; - // The exchange set puts a cell's files in the cell's own - // directory, so that directory names the owner. Both names - // must be COPIED: the walker reuses one buffer for the path, - // so a borrowed slice becomes the next entry's name. - const owner = a.dupe(u8, std.fs.path.basename(std.fs.path.dirname(entry.path) orelse "")) catch continue; - const name = a.dupe(u8, entry.path) catch continue; - aux_files.append(a, .{ .owner = owner, .name = name, .bytes = bytes }) catch {}; + // The text and pictures a cell references are the engine's to + // write, beside the archive it bakes; the walk only counts them. + if (auxfiles.isContent(entry.path)) { scan.files += 1; continue; } @@ -238,28 +229,24 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { const stem = std.fs.path.stem(std.fs.path.basename(cp)); // One directory per chart, as the exchange set does it: the archive // and the files that chart references travel together. - const chart_dir = std.fs.path.join(a, &.{ out_dir, stem }) catch continue; + // + // From an archive the output MIRRORS the entry's own path, so + // ENC_ROOT/US5MD12M/US5MD12M.000 becomes + // /ENC_ROOT/US5MD12M/US5MD12M.pmtiles. The exchange set + // already gives each cell a directory, so mirroring lands the + // referenced text beside the right chart — and two districts that + // share a boundary cell keep their own copies instead of one + // overwriting the other. + const chart_dir = if (zip_arc != null) + std.fs.path.join(a, &.{ out_dir, std.fs.path.dirname(cp) orelse "" }) catch continue + else + std.fs.path.join(a, &.{ out_dir, stem }) catch continue; std.Io.Dir.cwd().createDirPath(io, chart_dir) catch continue; const name = std.fmt.allocPrint(a, "{s}.pmtiles", .{stem}) catch continue; out_paths.append(a, std.fs.path.join(a, &.{ chart_dir, name }) catch continue) catch {}; } if (out_paths.items.len != cell_paths.items.len) return usageErr("out of memory naming archives"); - // The referenced text and pictures, beside the chart that names them. - for (cell_paths.items) |cp| { - const stem = std.fs.path.stem(std.fs.path.basename(cp)); - var mine = std.ArrayList(auxfiles.File).empty; - for (aux_files.items) |af| { - if (std.mem.eql(u8, af.owner, stem)) mine.append(a, af) catch {}; - } - if (mine.items.len == 0) continue; - const chart_dir = std.fs.path.join(a, &.{ out_dir, stem }) catch continue; - aux_written += auxfiles.writeDir(io, a, chart_dir, mine.items) catch |err| blk: { - std.debug.print("warning: aux files not written for {s} ({s})\n", .{ stem, @errorName(err) }); - break :blk 0; - }; - } - const n_workers = @min(workers, cell_paths.items.len); if (cell_paths.items.len > 1) { std.debug.print("baking {d} cell(s) across {d} worker(s)…\n", .{ cell_paths.items.len, n_workers }); @@ -269,7 +256,10 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { .total = @intCast(cell_paths.items.len), .paths = cell_paths.items, }; - const baked = chart.bakeChartsToFiles(io, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell); + const baked = if (zip_arc) |arc| + chart.bakeZipChartsToFiles(io, arc, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell, want_aux) + else + chart.bakeChartsToFiles(io, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell, want_aux); if (baked == 0) return usageErr("no cells baked (no .000 with M_COVR found)"); // bakeChartsToFiles reports a count, not which ones — a cell with no M_COVR @@ -306,10 +296,6 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { const part_path = try std.fs.path.join(a, &.{ out_dir, "partition.tpart" }); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = part_path, .data = part_bytes }); - if (aux_written > 0) { - std.debug.print(" {d} auxiliary file(s) beside their charts\n", .{aux_written}); - } - std.debug.print( "live structure -> {s}/\n {d} per-chart directory(s) + partition.tpart (serve z {d}..{d})\n", .{ out_dir, src.readers.len, src.minz, src.loop_max }, diff --git a/tools/common.zig b/tools/common.zig index 408aaaf0..8c6e43a1 100644 --- a/tools/common.zig +++ b/tools/common.zig @@ -142,7 +142,7 @@ pub fn printUsage() void { \\{s} — offline S-57 -> PMTiles baker / inspector \\ \\usage: - \\ tile57 bake -o [--rules DIR] [-j N] [--no-aux] + \\ tile57 bake -o [--rules DIR] [-j N] [--no-aux] \\ Produce a live-composite structure: bake each chart (a single .000 + \\ its auto-discovered updates, OR every .000 in an ENC_ROOT, at \\ native band scale) to its own directory, //.pmtiles, diff --git a/tools/main.zig b/tools/main.zig index 1ca46565..f83f11ec 100644 --- a/tools/main.zig +++ b/tools/main.zig @@ -1,7 +1,7 @@ //! tile57 — the offline S-57 -> PMTiles baker / inspector CLI. //! //! Subcommands: -//! bake -o [--rules DIR] [-j N] +//! bake -o [--rules DIR] [-j N] //! Bake each chart to /tiles/.pmtiles and write the ownership //! partition to /partition.tpart — the live-composite structure a //! runtime compositor serves tiles from on demand. A BSB/KAP sheet bakes the