From 96a60e5103036a9fe1e2abcb772bd67b5d8a8c2f Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Tue, 11 Aug 2026 13:22:37 -0400 Subject: [PATCH] zip: bake charts straight out of an archive Unzipping NOAA's All_ENCs.zip first costs 2.0 GiB of disk that is dead the moment the import ends. Instead the central directory is walked once and each cell is inflated when its turn comes, baked, and freed: 7,224 charts in about 12 minutes, never holding more than a worker's cells. What the engine can only read by path streams to disk at its own size, not twice it. tile57_zip_list, tile57_bake_zip_charts, tile57_bake_zip_rasters, tile57_zip_extract, and `tile57 bake archive.zip`. The bake now also writes the text and pictures a cell references beside its archive, from either source, when the chart has a directory of its own. Collecting them was the CLI's job, so nothing baked through the C ABI carried them and a pick report could name a caution note it could not show. --- build.zig | 15 ++ include/tile57.h | 71 ++++++ src/bake_root.zig | 1 + src/capi.zig | 183 ++++++++++++++- src/chart.zig | 173 +++++++++++++- src/root.zig | 3 +- src/zipsrc.zig | 578 ++++++++++++++++++++++++++++++++++++++++++++++ tools/bake.zig | 94 ++++---- tools/common.zig | 2 +- tools/main.zig | 2 +- 10 files changed, 1053 insertions(+), 69 deletions(-) create mode 100644 src/zipsrc.zig diff --git a/build.zig b/build.zig index 16c06d45..81798bce 100644 --- a/build.zig +++ b/build.zig @@ -484,7 +484,16 @@ pub fn build(b: *std.Build) void { // All pure packages, imported by name into engine / libtile57.a / the baker. // (portray is libc, wired separately into the lib + baker only.) + // Charts read straight out of a .zip, and the text/pictures a cell points + // at. Both are pure std and both are needed by the engine root AND by the + // separately-compiled chart module, so they must be modules: a relative + // import from each would put one file in two modules. + const zipsrc_mod = b.addModule("zipsrc", .{ .root_source_file = b.path("src/zipsrc.zig") }); + const auxfiles_mod = b.addModule("auxfiles", .{ .root_source_file = b.path("src/auxfiles.zig") }); + const pure_pkgs = [_]std.Build.Module.Import{ + .{ .name = "zipsrc", .module = zipsrc_mod }, + .{ .name = "auxfiles", .module = auxfiles_mod }, .{ .name = "s57", .module = s57_mod }, .{ .name = "s101", .module = s101_mod }, .{ .name = "tiles", .module = tiles_mod }, @@ -681,6 +690,8 @@ pub fn build(b: *std.Build) void { .{ .name = "portray", .module = portray_mod }, .{ .name = "sprite", .module = sprite_mod }, .{ .name = "catalog", .module = catalog_embed }, + .{ .name = "zipsrc", .module = zipsrc_mod }, + .{ .name = "auxfiles", .module = auxfiles_mod }, }, }); chart_mod.addImport("style", style_mod); // linestyle XML analysis @@ -837,6 +848,10 @@ pub fn build(b: *std.Build) void { }); _ = addPkgTest(b, test_step, "src/style/style.zig", target, optimize, &.{}); _ = addPkgTest(b, test_step, "src/errors.zig", target, optimize, &.{}); + // Charts read straight out of a .zip, and the aux files that travel with + // them: pure over std, so each tests alone. + _ = addPkgTest(b, test_step, "src/zipsrc.zig", target, optimize, &.{}); + _ = addPkgTest(b, test_step, "src/auxfiles.zig", target, optimize, &.{}); // Geometry core for the cross-band composition. No longer std-only: plane.zig // reads its partition tuning/stats valves via std.c.getenv, so the test binary // needs libc for the same reason compose's does, below. diff --git a/include/tile57.h b/include/tile57.h index 43f8a1a9..67b02243 100644 --- a/include/tile57.h +++ b/include/tile57.h @@ -290,6 +290,77 @@ tile57_status tile57_bake_rasters(const char *const *in_paths, const char *const tile57_bake_progress progress, tile57_bake_label label, void *progress_ctx, uint32_t *out_baked, tile57_error *err); +/* ---- charts inside a .zip --------------------------------------------------- + * + * A chart archive arrives as one download and is mostly not the file you want: + * NOAA's All_ENCs.zip is 788 MB of deflate holding 2.0 GiB across 27,680 + * entries. Unzipping it costs the mariner 2.0 GiB of disk that is dead the + * moment the import ends, on top of the charts they keep. These calls read the + * charts out of the archive where they lie: each cell is inflated when its turn + * comes, baked, and freed, so peak memory is the worker count times one chart + * and no intermediate copy is ever written. + * + * Every call opens the archive, works, and closes it. Walking the central + * directory of that 27,680-entry archive costs about 8 ms, so there is no + * handle to hold, close, or keep off another thread. + * + * `names` are entry names exactly as tile57_zip_list reports them. Destinations + * are always the CALLER's `out_paths`, never a name from the archive, so an + * entry called "../../etc/rc" cannot choose where anything lands. */ + +/* List what a .zip holds, into *out / *out_len (free with tile57_free), as + * [{"name":..,"size":..,"packed":..}, ..] in central-directory order. Sizes are + * uncompressed and compressed bytes. Directory entries are omitted. The buffer + * is NUL-terminated past *out_len, so it reads as a C string too. + * + * This says what is IN the archive, not what is a chart: classification is the + * host's, and only the host knows which formats it supports. */ +tile57_status tile57_zip_list(const char *zip_path, uint8_t **out, size_t *out_len, + tile57_error *err); + +/* tile57_bake_files, reading each cell straight out of `zip_path`: names[i] is + * a .000 entry, out_paths[i] is where its PMTiles goes. Its .001.. updates are + * found and applied from the archive, by the same rule the on-disk bake uses on + * a cell's directory: sequential from .001, stopping at the first gap. + * + * The text and pictures a cell references (TXTDSC, PICREP) are written beside + * the archive with an index.json manifest, so a pick report can resolve them + * through tile57_aux_open — but ONLY when out_paths[i] puts the archive in a + * directory named for the chart (/US1EEZ3M/US1EEZ3M.pmtiles). Those files + * are named per exchange set rather than per chart, so charts baked flat into + * one directory would overwrite each other's manifests; rather than guess, the + * rule is the exchange set's own shape. Same contract otherwise — caller-owned + * lists, `label` names a finished chart by index, `progress` may cancel, + * *out_baked counts what was written. A name the archive does not hold is + * skipped, like a cell that fails to bake. */ +tile57_status tile57_bake_zip_charts(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + +/* tile57_bake_rasters, reading each BSB/KAP sheet straight out of `zip_path`. + * Same contract as tile57_bake_rasters, including that `workers` is a memory + * bound: a sheet is decoded whole whether it came from a file or an archive. */ +tile57_status tile57_bake_zip_rasters(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + +/* Inflate names[i] out of `zip_path` to out_paths[i], streaming: a 32 KiB + * window, whatever the file weighs. This is the path for what the engine can + * only read BY PATH — an .mbtiles or .pmtiles it memory-maps — where the file + * must exist on disk but a second copy of it need not. A 4 GiB .mbtiles + * therefore costs 4 GiB, not 8 GiB. + * + * Serial by design: these are the big entries, the disk is the limit, and + * parallel writers only contend for it. `progress` fires after each file and + * may cancel; *out_done counts what was written. An entry the archive does not + * hold is skipped. */ +tile57_status tile57_zip_extract(const char *zip_path, const char *const *names, + const char *const *out_paths, size_t n, + tile57_bake_progress progress, void *progress_ctx, + uint32_t *out_done, tile57_error *err); + /* Read a PMTiles archive's metadata JSON blob (decompressed) into *out / * *out_len (free with tile57_free); NULL/0 when the archive carries none. A * per-chart bake embeds the chart's M_COVR coverage + cscl + date/name under a diff --git a/src/bake_root.zig b/src/bake_root.zig index 01b19f7f..f6eae77b 100644 --- a/src/bake_root.zig +++ b/src/bake_root.zig @@ -29,6 +29,7 @@ pub const s101_adapter = root.s101_adapter; pub const catalogue = root.catalogue; pub const bake_enc = root.bake_enc; pub const auxfiles = root.auxfiles; +pub const zipsrc = root.zipsrc; // charts read straight out of a .zip pub const geometry = @import("geometry"); // integer geometry: boolean, plane, partition pub const portray = @import("portray"); diff --git a/src/capi.zig b/src/capi.zig index 0342425c..d3e615e9 100644 --- a/src/capi.zig +++ b/src/capi.zig @@ -17,6 +17,7 @@ const mariner = @import("style").mariner; const style = @import("style"); const errors = @import("errors"); // the engine error taxonomy + describe() const raster = @import("raster"); // raster charts (tile57_raster_chart_*) +const zipsrc = @import("zipsrc"); // charts read straight out of a .zip // The S-52 ColorProfiles/colorProfile.xml baked into the library (build.zig), so // the style C ABI generates colortables + a base style template with no on-disk // catalogue. Symbols/linestyles are NOT embedded here (only the bake exe needs them). @@ -412,7 +413,7 @@ export fn tile57_bake_files( in_list[i] = std.mem.span(ins[i]); out_list[i] = std.mem.span(outs[i]); } - const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label); + const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label, true); if (out_baked) |p| p.* = @intCast(baked); return OK; } @@ -422,9 +423,12 @@ const RasterJob = struct { next: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), done: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), cancel: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + /// Sheet paths, or — when `zip` is set — names of entries inside it. in: []const []const u8, out: []const []const u8, ok: []bool, + /// The archive the sheets are read out of, or null to read from disk. + zip: ?*const zipsrc.Archive = null, progress: chart.BakeProgress, label: chart.BakeLabel, ctx: ?*anyopaque, @@ -436,7 +440,14 @@ const RasterJob = struct { fn bakeOneRaster(io: std.Io, job: *RasterJob, i: usize) void { const a = std.heap.c_allocator; const stem = std.fs.path.stem(std.fs.path.basename(job.in[i])); - const kap = std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return; + // From the archive or from disk — the warp below cannot tell which. + const kap = blk: { + if (job.zip) |z| { + const idx = z.find(job.in[i]) orelse return; + break :blk z.readAlloc(a, io, idx, MAX_RASTER_BYTES) catch return; + } + break :blk std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return; + }; defer a.free(kap); const baked = raster.bakebsb.bakeBytes(a, kap, stem) catch return; defer a.free(baked.bytes); @@ -584,6 +595,174 @@ fn writeRasterPartition(out_list: []const []const u8, ok: []const bool) void { std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }) catch {}; } +// ---- charts inside a .zip ---------------------------------------------------- +// A chart archive is opened, read from, and closed within each call. Walking a +// 27,680-entry central directory costs about 8 ms, so the alternative — an open +// handle the host must hold, close, and keep off other threads — buys nothing +// and can be got wrong. + +/// The largest a single raster sheet may claim to expand to. A KAP runs to tens +/// of megabytes; the cap is here so a bad header fails instead of allocating. +const MAX_RASTER_BYTES: u64 = 512 << 20; + +/// List what a .zip holds: [{"name":..,"size":..,"packed":..}, ..] into +/// *out / *out_len (free with tile57_free). See tile57.h. +export fn tile57_zip_list(zip_path: ?[*:0]const u8, out: ?*?[*]u8, out_len: ?*usize, err: ?*CError) callconv(.c) c_int { + const o, const n = bytesOut(out, out_len) catch return failWith(err, .badarg, bad_out); + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + const json = arc.toJson(gpa) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(json); + // Copied with its terminator: the payload is length-delimited by *out_len + // AND readable as a C string, so a host can take either. + const p = exportAlloc(json.len + 1) orelse return failWith(err, .nomem, "out of memory"); + @memcpy(p[0 .. json.len + 1], json[0 .. json.len + 1]); + o.* = p; + n.* = json.len; + return OK; +} + +/// Inflate entries out of a .zip to paths the CALLER names, streaming. See tile57.h. +export fn tile57_zip_extract( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + progress: chart.BakeProgress, + progress_ctx: ?*anyopaque, + out_done: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_done) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + + // Serial on purpose: this is one file stream to disk per entry, and the + // entries that come this way are the big ones (a 4 GiB .mbtiles), where + // the disk is the limit and parallel writers only fight over it. + var done: u32 = 0; + for (0..n) |i| { + const name = std.mem.span(ns[i]); + const idx = arc.find(name) orelse continue; + arc.extractTo(io, idx, std.mem.span(outs[i])) catch continue; + done += 1; + if (progress) |cb| { + if (!cb(progress_ctx, @intCast(i + 1), @intCast(n))) break; + } + } + if (out_done) |p| p.* = done; + return OK; +} + +/// tile57_bake_files reading the cells STRAIGHT OUT of a .zip. See tile57.h. +export fn tile57_bake_zip_charts( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + for (0..n) |i| { + in_list[i] = std.mem.span(ns[i]); + out_list[i] = std.mem.span(outs[i]); + } + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + const baked = chart.bakeZipChartsToFiles(io, &arc, in_list, out_list, null, workers, progress, progress_ctx, label, true); + if (out_baked) |p| p.* = @intCast(baked); + return OK; +} + +/// tile57_bake_rasters reading the sheets STRAIGHT OUT of a .zip. See tile57.h. +export fn tile57_bake_zip_rasters( + zip_path: ?[*:0]const u8, + names: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null"); + const ns = names orelse return failWith(err, .badarg, "names must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + const ok = gpa.alloc(bool, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(ok); + @memset(ok, false); + for (0..n) |i| { + in_list[i] = std.mem.span(ns[i]); + out_list[i] = std.mem.span(outs[i]); + } + + const io = sharedIo(); + var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp); + defer arc.deinit(); + + var job = RasterJob{ + .in = in_list, + .out = out_list, + .ok = ok, + .zip = &arc, + .progress = progress, + .label = label, + .ctx = progress_ctx, + }; + // Same deep stacks as tile57_bake_rasters: the warp is the same work, + // only the bytes arrive from the archive instead of a file. + const stack = 16 * 1024 * 1024; + var threads: [8]std.Thread = undefined; + const want = @min(@max(workers, 1), @min(threads.len, n)); + var spawned: usize = 0; + while (spawned < want) : (spawned += 1) { + threads[spawned] = std.Thread.spawn(.{ .stack_size = stack }, rasterWorker, .{&job}) catch break; + } + if (spawned == 0) rasterWorker(&job); + for (threads[0..spawned]) |t| t.join(); + + var baked: u32 = 0; + for (ok) |o| { + if (o) baked += 1; + } + if (out_baked) |p| p.* = baked; + if (baked > 0) writeRasterPartition(out_list, ok); + return OK; +} + /// The metadata JSON blob of a PMTiles archive (decompressed) — e.g. the embedded /// per-cell "coverage" a single-cell bake carries — into *out / *out_len (free with /// tile57_free); NULL/0 when the archive carries none. See tile57.h. diff --git a/src/chart.zig b/src/chart.zig index 9c3378f3..07c747e5 100644 --- a/src/chart.zig +++ b/src/chart.zig @@ -35,6 +35,8 @@ const embedded_assets = @import("catalog"); // S-101 portrayal assets (renderVie const style = @import("style"); // displayDenomZ (the physical display-scale formula) const cell_coverage = @import("coverage"); // per-cell M_COVR coverage embedded in archive metadata const compose_mod = @import("compose"); // the runtime compositor (compose-backed view renders) +const zipsrc = @import("zipsrc"); // charts read straight out of a .zip +const auxfiles = @import("auxfiles"); // the text and pictures a cell points at // c_allocator, not smp_allocator: smp's per-CPU slab freelists never return // pages to the OS, so a long-lived host process's footprint ratchets up to the @@ -715,11 +717,13 @@ fn openCell(bytes: []const u8, rules_dir: ?[]const u8) ?*Chart { return src; } -/// A single ENC cell's on-disk bytes: base .000 + its sequential .001.. update chain. -const CellFiles = struct { +/// A single ENC cell's bytes: base .000 + its sequential .001.. update chain. +/// Where they came from is not recorded, so a cell read out of a directory and +/// one inflated out of a zip bake through the same path. +pub const CellFiles = struct { base: []u8, updates: [][]u8, - fn deinit(self: *CellFiles) void { + pub fn deinit(self: *CellFiles) void { gpa.free(self.base); for (self.updates) |u| gpa.free(u); gpa.free(self.updates); @@ -763,6 +767,99 @@ fn readCellFiles(path: []const u8) !CellFiles { return .{ .base = base, .updates = try updates.toOwnedSlice(gpa) }; } +/// The most a single ENC cell may claim to expand to. Cells run to a few MiB; +/// a header promising more than this is a damaged or hostile archive, and the +/// point of a cap is to find that out before allocating rather than after. +pub const MAX_CELL_BYTES: u64 = 256 << 20; + +/// `readCellFiles` out of a zip: the base entry plus its update chain, each +/// inflated on its own. Nothing is written to disk and nothing larger than one +/// cell is held. +pub fn readCellFromZip(io: std.Io, arc: *const zipsrc.Archive, idx: usize) !CellFiles { + const base = try arc.readAlloc(gpa, io, idx, MAX_CELL_BYTES); + errdefer gpa.free(base); + + const up_idx = try arc.updatesFor(gpa, idx); + defer gpa.free(up_idx); + var updates = std.ArrayList([]u8).empty; + errdefer { + for (updates.items) |u| gpa.free(u); + updates.deinit(gpa); + } + // A broken update stops the chain rather than the cell: an ENC applied + // through update 3 is a chart, and refusing to draw it because update 4 + // is corrupt leaves the mariner with nothing. + for (up_idx) |ui| { + const ub = arc.readAlloc(gpa, io, ui, MAX_CELL_BYTES) catch break; + updates.append(gpa, ub) catch { + gpa.free(ub); + break; + }; + } + return .{ .base = base, .updates = try updates.toOwnedSlice(gpa) }; +} + +/// The directory to write a chart's referenced files into, or null when the +/// caller did not give this chart a directory of its own. +/// +/// A cell's .TXT and pictures are named per exchange set, not per chart — +/// US1EEZ3M references US1EEZ3A.TXT — so several charts baked flat into one +/// directory would share a manifest and overwrite each other's. Rather than +/// guess, the rule is explicit: aux files are written only when the archive +/// sits in a directory named for the chart (/US1EEZ3M/US1EEZ3M.pmtiles), +/// which is the exchange set's own shape and what tile57_aux_open expects. +fn auxDirFor(out_path: []const u8, stem: []const u8) ?[]const u8 { + const dir = std.fs.path.dirname(out_path) orelse return null; + if (!std.mem.eql(u8, std.fs.path.basename(dir), stem)) return null; + return dir; +} + +/// Write the text and pictures a cell references beside its baked archive, out +/// of the cell's own directory in the archive. Best-effort: a chart still +/// draws without its caution notes, so a failure here is not a bake failure. +fn writeAuxFromZip(io: std.Io, arc: *const zipsrc.Archive, idx: usize, out_path: []const u8) void { + const stem = std.fs.path.stem(std.fs.path.basename(arc.entries[idx].name)); + const dst = auxDirFor(out_path, stem) orelse return; + + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + var files = std.ArrayList(auxfiles.File).empty; + for (arc.siblingsOf(idx)) |si| { + const name = arc.entries[si].name; + if (!auxfiles.isContent(name)) continue; + const bytes = arc.readAlloc(a, io, si, MAX_CELL_BYTES) catch continue; + files.append(a, .{ .owner = stem, .name = name, .bytes = bytes }) catch continue; + } + _ = auxfiles.writeDir(io, a, dst, files.items) catch {}; +} + +/// The same, for a cell read from a directory: its referenced files are the +/// aux content sitting beside it. +fn writeAuxFromDir(io: std.Io, cell_path: []const u8, out_path: []const u8) void { + const stem = std.fs.path.stem(std.fs.path.basename(cell_path)); + const dst = auxDirFor(out_path, stem) orelse return; + + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + const src_dir = std.fs.path.dirname(cell_path) orelse "."; + var dir = std.Io.Dir.cwd().openDir(io, src_dir, .{ .iterate = true }) catch return; + defer dir.close(io); + + var files = std.ArrayList(auxfiles.File).empty; + var it = dir.iterate(); + while (it.next(io) catch null) |ent| { + if (ent.kind != .file or !auxfiles.isContent(ent.name)) continue; + // The iterator reuses its name buffer, so both the name and the bytes + // must be copied before the next step. + const name = a.dupe(u8, ent.name) catch continue; + const bytes = dir.readFileAlloc(io, name, a, .unlimited) catch continue; + files.append(a, .{ .owner = stem, .name = name, .bytes = bytes }) catch continue; + } + _ = auxfiles.writeDir(io, a, dst, files.items) catch {}; +} + /// Bake a SINGLE .000 cell (+ updates) to a PMTiles archive over its NATIVE band's /// zoom range (`bandZooms(bandOf(cscl))`) and nothing else — the composite model bakes /// each cell at its own compilation scale; the stitcher combines them and handles any @@ -773,13 +870,22 @@ fn readCellFiles(path: []const u8) !CellFiles { /// so the composite stitcher rebuilds the ownership partition from the baked archives /// without re-parsing the .000. Read it back with `decodedCoverageFromArchive`. pub fn bakeChartBytes(cell_path: []const u8, rules_dir: ?[]const u8) !?[]u8 { + var cf = try readCellFiles(cell_path); + defer cf.deinit(); + return bakeCellFiles(&cf, cell_path, rules_dir); +} + +/// `bakeChartBytes` for a cell ALREADY IN MEMORY. `cell_name` is the cell's +/// name — a path or a zip entry name; only its stem is read, as the ownership +/// tie-break and the pick report's source-cell badge. Nothing is read from +/// disk, so this is the entry point for a cell inflated straight out of an +/// archive. +pub fn bakeCellFiles(cf: *const CellFiles, cell_name: []const u8, rules_dir: ?[]const u8) !?[]u8 { // Populate the read-only portrayal globals (feature catalogue + complex-linestyle table) // before portraying: without them, complex lines fall back to plain geometry and their S-52 // linestyle is dropped from the tile. Idempotent; in the parallel batch path bakeChartsParallel // has already warmed up before spawning workers, so this is a no-op there (and race-free). warmup(); - var cf = try readCellFiles(cell_path); - defer cf.deinit(); // Capture coverage for the embedded sidecar (one cheap parse). The stem is the // ownership tie-break name — matches the coverage loader. @@ -792,7 +898,7 @@ pub fn bakeChartBytes(cell_path: []const u8, rules_dir: ?[]const u8) !?[]u8 { // The dataset stem is the ownership tie-break name AND the pick-report's // "source cell" badge — pass it into the tile bake below (bakeArchive borrows // it for cell.name), or every feature's `cell` prop bakes empty. - const stem = std.fs.path.stem(std.fs.path.basename(cell_path)); + const stem = std.fs.path.stem(std.fs.path.basename(cell_name)); var cscl: i32 = s57.peekScale(gpa, cf.base) orelse 0; if (parseAnyCell(cf.base, cf.updates)) |loaded| { var cell = loaded.cell; @@ -888,9 +994,14 @@ pub const BakeLabel = ?*const fn (?*anyopaque, u32) callconv(.c) void; const BakeFileCtx = struct { next: std.atomic.Value(usize), + /// Cell paths, or — when `zip` is set — names of entries inside it. in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, + /// The archive the cells are read out of, or null to read from disk. The + /// archive is immutable once opened and every read opens its own handle, + /// so all the workers share this one. + zip: ?*const zipsrc.Archive = null, io: std.Io, ok: []bool, ms: []i64, // per-cell wall time — the bake profiles itself (slowest cells printed at the end) @@ -900,6 +1011,8 @@ const BakeFileCtx = struct { done: std.atomic.Value(u32), /// Set when a progress callback returned false; every worker drains out at its next cell. cancel: std.atomic.Value(bool), + /// Write the text and pictures each cell references beside its archive. + aux: bool = true, }; fn bakeOneToFile(ctx: *BakeFileCtx, i: usize) void { @@ -908,9 +1021,30 @@ fn bakeOneToFile(ctx: *BakeFileCtx, i: usize) void { const t1 = std.Io.Clock.awake.now(ctx.io); ctx.ms[i] = @intCast(@divTrunc(t1.nanoseconds - t0.nanoseconds, 1_000_000)); } - const arc = (bakeChartBytes(ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + // From the archive or from disk — the bake below cannot tell which. + var zip_idx: ?usize = null; + const arc = blk: { + if (ctx.zip) |z| { + const idx = z.find(ctx.in_paths[i]) orelse return; + zip_idx = idx; + var cf = readCellFromZip(ctx.io, z, idx) catch return; + defer cf.deinit(); + break :blk (bakeCellFiles(&cf, ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + } + break :blk (bakeChartBytes(ctx.in_paths[i], ctx.rules_dir) catch null) orelse return; + }; defer freeBytes(arc); std.Io.Dir.cwd().writeFile(ctx.io, .{ .sub_path = ctx.out_paths[i], .data = arc }) catch return; + // The files this cell references, beside it — a pick report resolves + // TXTDSC by name, so a chart without them answers "see US1EEZ3A.TXT" and + // cannot show it. + if (ctx.aux) { + if (zip_idx) |idx| { + writeAuxFromZip(ctx.io, ctx.zip.?, idx, ctx.out_paths[i]); + } else { + writeAuxFromDir(ctx.io, ctx.in_paths[i], ctx.out_paths[i]); + } + } var digest: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(arc, &digest, .{}); const hex = std.fmt.bytesToHex(digest, .lower); @@ -945,7 +1079,26 @@ fn bakeFileWorker(ctx: *BakeFileCtx) void { /// after each cell and may CANCEL by returning false (see BakeProgress); `label(progress_ctx, i)` /// fires beside it and names the chart that finished. Race-free (warms up first; each bake is /// independent). Returns the count written — fewer than in_paths.len when cancelled. -pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel) usize { +/// +/// `aux` writes the text and pictures each cell references beside its archive +/// (see auxDirFor for when that is possible). +pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { + return bakeToFiles(io, null, in_paths, out_paths, rules_dir, workers, progress, progress_ctx, label, aux); +} + +/// `bakeChartsToFiles` reading every cell STRAIGHT OUT of `arc`: `names` are +/// entry names inside the archive rather than paths, and nothing is unzipped +/// — each cell and its updates are inflated when their turn comes and freed +/// with the archive they bake into. Workers pull from one open archive +/// concurrently, so this runs at the same rate as the on-disk bake. +/// +/// A name the archive does not hold is skipped, like a cell that fails to +/// bake: *out_baked counts what was written. +pub fn bakeZipChartsToFiles(io: std.Io, arc: *const zipsrc.Archive, names: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { + return bakeToFiles(io, arc, names, out_paths, rules_dir, workers, progress, progress_ctx, label, aux); +} + +fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []const u8, out_paths: []const []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel, aux: bool) usize { std.debug.assert(out_paths.len == in_paths.len); if (in_paths.len == 0) return 0; warmup(); @@ -955,7 +1108,7 @@ pub fn bakeChartsToFiles(io: std.Io, in_paths: []const []const u8, out_paths: [] const cell_ms = gpa.alloc(i64, in_paths.len) catch return 0; defer gpa.free(cell_ms); @memset(cell_ms, 0); - var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false) }; + var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux }; var n = @min(@max(workers, 1), in_paths.len); if (n > MAX_BAKE_WORKERS) n = MAX_BAKE_WORKERS; if (n <= 1) { @@ -1038,7 +1191,7 @@ pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: out_paths.append(a, out_path) catch continue; } if (in_paths.items.len == 0) return 0; - return bakeChartsToFiles(io, in_paths.items, out_paths.items, rules_dir, workers, progress, progress_ctx, label); + return bakeChartsToFiles(io, in_paths.items, out_paths.items, rules_dir, workers, progress, progress_ctx, label, true); } /// The file's modification time in nanoseconds, or null if it doesn't exist / can't be statted. diff --git a/src/root.zig b/src/root.zig index ad049904..5012c3a6 100644 --- a/src/root.zig +++ b/src/root.zig @@ -23,7 +23,8 @@ pub const catalogue = s101.catalogue; pub const bake_enc = @import("scene").bake_enc; // banded multi-cell ENC_ROOT -> PMTiles pub const style = @import("style"); // colortables, line styles, and style.json generation pub const mariner = @import("style").mariner; // mariner-driven MapLibre style patching -pub const auxfiles = @import("auxfiles.zig"); // the text and pictures a cell points at +pub const auxfiles = @import("auxfiles"); // the text and pictures a cell points at +pub const zipsrc = @import("zipsrc"); // charts read straight out of a .zip // capi (the C ABI) lives in lib_root.zig so the test/bake exes stay pure Zig. test { diff --git a/src/zipsrc.zig b/src/zipsrc.zig new file mode 100644 index 00000000..566142b3 --- /dev/null +++ b/src/zipsrc.zig @@ -0,0 +1,578 @@ +//! Charts read STRAIGHT OUT of a .zip, one entry at a time. +//! +//! A chart archive is big: NOAA's All_ENCs.zip is 788 MB of deflate holding +//! 2.0 GiB across 27,680 entries — 7,224 cells, their 4,414 updates, and +//! 15,539 .TXT the pick report resolves by name. Unzipping it first costs the +//! mariner 2.0 GiB of disk that is dead the moment the bake ends, on top of +//! the baked charts they actually keep. So nothing here unzips: the central +//! directory is walked once (2 ms for those 27,680 entries), and each file is +//! inflated on its own, when it is needed, into a buffer its own size. +//! +//! Two properties make that work, and both are load-bearing: +//! +//! * Entries are randomly addressable — seek to the local header, inflate. +//! A read does not disturb any other read, so `readAlloc` opens its own +//! handle and the parallel bake keeps every core busy pulling different +//! cells out of one archive. There is no shared cursor to lock. +//! * Inflating is bounded — a 32 KiB window plus the read buffer, whatever +//! the entry's size. `extractTo` therefore streams a 4 GiB .mbtiles to +//! disk in constant memory, and needs no space for a second copy. +//! +//! Nothing here decides WHAT a chart is. The archive reports its names and +//! reads whichever ones it is asked for; the host classifies. What it does +//! know is the exchange set's SHAPE, because that is structure rather than +//! policy: a cell's .001.. updates belong to that cell (`updatesFor`), and +//! the files sitting in a cell's own directory are the ones it references +//! (`siblingsOf`) — the .TXT a TXTDSC names, the picture a PICREP names. + +const std = @import("std"); +const zip = std.zip; +const flate = std.compress.flate; +const Allocator = std.mem.Allocator; + +pub const Error = error{ + BadLocalHeader, + UnsupportedCompressionMethod, + EntryTooLarge, +}; + +pub const Entry = struct { + /// The name as stored, e.g. "ENC_ROOT/US5MD12M/US5MD12M.000". Arena-owned. + name: []const u8, + uncompressed_size: u64, + compressed_size: u64, + raw: zip.Iterator.Entry, +}; + +/// A zip opened for reading charts. Read-only once opened, so any number of +/// threads may `readAlloc`/`extractTo` from one `Archive` at the same time. +pub const Archive = struct { + gpa: Allocator, + arena: std.heap.ArenaAllocator, + /// Our own copy of the archive's path: each read opens the file again + /// rather than sharing a cursor. + path: []const u8, + entries: []Entry, + by_name: std.StringHashMapUnmanaged(usize), + /// Directory prefix (no trailing '/') -> the entries directly in it. Built + /// once, because the alternative is rescanning 27,680 names per cell. + by_dir: std.StringHashMapUnmanaged([]usize), + + /// Walk the central directory. Directory entries (trailing '/') are left + /// out — they are never charts. Encrypted and multi-disk archives are + /// rejected here, by `zip.Iterator`, rather than at the first read. + pub fn open(gpa: Allocator, io: std.Io, path: []const u8) !Archive { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const a = arena.allocator(); + + const f = try std.Io.Dir.cwd().openFile(io, path, .{}); + defer f.close(io); + var rbuf: [64 * 1024]u8 = undefined; + var fr = f.reader(io, &rbuf); + + var it = try zip.Iterator.init(&fr); + var entries: std.ArrayList(Entry) = .empty; + errdefer entries.deinit(gpa); + try entries.ensureTotalCapacity(gpa, @intCast(@min(it.cd_record_count, 1 << 20))); + + var name_buf: [4096]u8 = undefined; + while (try it.next()) |e| { + if (e.filename_len == 0 or e.filename_len > name_buf.len) continue; + const name = name_buf[0..e.filename_len]; + try fr.seekTo(e.header_zip_offset + @sizeOf(zip.CentralDirectoryFileHeader)); + try fr.interface.readSliceAll(name); + if (name[name.len - 1] == '/') continue; // a directory, not a chart + try entries.append(gpa, .{ + .name = try a.dupe(u8, name), + .uncompressed_size = e.uncompressed_size, + .compressed_size = e.compressed_size, + .raw = e, + }); + } + + const owned = try entries.toOwnedSlice(gpa); + errdefer gpa.free(owned); + var by_name: std.StringHashMapUnmanaged(usize) = .empty; + errdefer by_name.deinit(gpa); + try by_name.ensureTotalCapacity(gpa, @intCast(owned.len)); + for (owned, 0..) |e, i| by_name.putAssumeCapacity(e.name, i); + + // Group by directory in one pass, into the arena, so `siblingsOf` is a + // lookup rather than a scan. + var grow: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(usize)) = .empty; + defer { + var it2 = grow.valueIterator(); + while (it2.next()) |v| v.deinit(gpa); + grow.deinit(gpa); + } + for (owned, 0..) |e, i| { + const d = dirNameOf(e.name); + const slot = try grow.getOrPut(gpa, d); + if (!slot.found_existing) slot.value_ptr.* = .empty; + try slot.value_ptr.append(gpa, i); + } + var by_dir: std.StringHashMapUnmanaged([]usize) = .empty; + errdefer by_dir.deinit(gpa); + try by_dir.ensureTotalCapacity(gpa, grow.count()); + var git = grow.iterator(); + while (git.next()) |kv| { + by_dir.putAssumeCapacity(kv.key_ptr.*, try a.dupe(usize, kv.value_ptr.items)); + } + + // Every arena allocation must happen BEFORE the literal below copies + // the arena: the copy freezes the buffer list, so anything allocated + // while building the literal lands in a node the copy cannot free. + const path_copy = try a.dupe(u8, path); + return .{ + .gpa = gpa, + .arena = arena, + .path = path_copy, + .entries = owned, + .by_name = by_name, + .by_dir = by_dir, + }; + } + + pub fn deinit(self: *Archive) void { + self.by_name.deinit(self.gpa); + self.by_dir.deinit(self.gpa); + self.gpa.free(self.entries); + self.arena.deinit(); + } + + pub fn find(self: *const Archive, name: []const u8) ?usize { + return self.by_name.get(name); + } + + /// Inflate entry `i` into a fresh buffer (caller frees). `max` bounds what + /// the archive is allowed to claim an entry expands to — an ENC cell is a + /// few MiB, so a header promising gigabytes is a reason to stop, not to + /// allocate. Opens its own file handle: safe to call from many threads. + pub fn readAlloc(self: *const Archive, gpa: Allocator, io: std.Io, i: usize, max: u64) ![]u8 { + const e = self.entries[i]; + if (e.uncompressed_size > max) return Error.EntryTooLarge; + const buf = try gpa.alloc(u8, @intCast(e.uncompressed_size)); + errdefer gpa.free(buf); + + var w: std.Io.Writer = .fixed(buf); + try self.streamEntry(io, i, &w); + if (w.end != buf.len) return error.EndOfStream; + return buf; + } + + /// Inflate entry `i` straight to `out_path`, in constant memory however + /// large it is — the one way a 4 GiB .mbtiles lands on disk without a + /// second copy of it existing anywhere. `out_path` is the CALLER's name + /// for the file, never the archive's, so a hostile entry name cannot + /// choose where this writes. + pub fn extractTo(self: *const Archive, io: std.Io, i: usize, out_path: []const u8) !void { + const out = try std.Io.Dir.cwd().createFile(io, out_path, .{ .truncate = true }); + defer out.close(io); + var wbuf: [256 * 1024]u8 = undefined; + var fw = out.writer(io, &wbuf); + try self.streamEntry(io, i, &fw.interface); + try fw.interface.flush(); + } + + /// The `.001..` update chain belonging to the `.000` cell at `i`, in order, + /// stopping at the first gap — the same rule the on-disk reader applies to + /// a cell's directory, applied to the archive's names instead. Empty when + /// `i` is not a base cell. Caller frees. + pub fn updatesFor(self: *const Archive, gpa: Allocator, i: usize) ![]usize { + var out: std.ArrayList(usize) = .empty; + errdefer out.deinit(gpa); + const name = self.entries[i].name; + if (name.len < 4 or !std.mem.eql(u8, name[name.len - 4 ..], ".000")) { + return out.toOwnedSlice(gpa); + } + const stem = name[0 .. name.len - 4]; + var buf: [4096]u8 = undefined; + var u: u32 = 1; + while (u <= 999) : (u += 1) { + const up = std.fmt.bufPrint(&buf, "{s}.{d:0>3}", .{ stem, u }) catch break; + const idx = self.find(up) orelse break; + try out.append(gpa, idx); + } + return out.toOwnedSlice(gpa); + } + + /// Every entry in the same directory as `i`, `i` included. The exchange + /// set puts a cell's referenced text and pictures in the cell's own + /// directory, so this is the candidate list an aux-file pass filters — + /// the archive does not decide which of them are content, which is why it + /// does not drop the cell either. Borrowed; valid until deinit. + pub fn siblingsOf(self: *const Archive, i: usize) []const usize { + return self.by_dir.get(dirNameOf(self.entries[i].name)) orelse &.{}; + } + + /// Every entry as JSON: [{"name":..,"size":..,"packed":..}, ..]. The host + /// classifies from this; the archive takes no view on which are charts. + /// NUL-terminated so it can cross the C ABI as a plain string. + pub fn toJson(self: *const Archive, gpa: Allocator) ![:0]u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(gpa); + try buf.append(gpa, '['); + for (self.entries, 0..) |e, i| { + if (i != 0) try buf.append(gpa, ','); + try buf.appendSlice(gpa, "{\"name\":"); + try appendJsonString(gpa, &buf, e.name); + try buf.print(gpa, ",\"size\":{d},\"packed\":{d}}}", .{ e.uncompressed_size, e.compressed_size }); + } + try buf.append(gpa, ']'); + return buf.toOwnedSliceSentinel(gpa, 0); + } + + /// Where entry `i`'s compressed bytes start: past its LOCAL header, whose + /// name and extra fields may be sized differently from the central + /// directory's copy. + fn dataOffset(e: zip.Iterator.Entry, fr: *std.Io.File.Reader) !u64 { + try fr.seekTo(e.file_offset); + const lh = try fr.interface.takeStruct(zip.LocalFileHeader, .little); + if (!std.mem.eql(u8, &lh.signature, &zip.local_file_header_sig)) return Error.BadLocalHeader; + return e.file_offset + @sizeOf(zip.LocalFileHeader) + lh.filename_len + lh.extra_len; + } + + /// Inflate one entry into `w`. Bounded: a 32 KiB window and the read + /// buffer, whatever the entry weighs. + fn streamEntry(self: *const Archive, io: std.Io, i: usize, w: *std.Io.Writer) !void { + const e = self.entries[i].raw; + const f = try std.Io.Dir.cwd().openFile(io, self.path, .{}); + defer f.close(io); + var rbuf: [64 * 1024]u8 = undefined; + var fr = f.reader(io, &rbuf); + try fr.seekTo(try dataOffset(e, &fr)); + switch (e.compression_method) { + .store => try fr.interface.streamExact64(w, e.uncompressed_size), + .deflate => { + var window: [flate.max_window_len]u8 = undefined; + var d: flate.Decompress = .init(&fr.interface, .raw, &window); + try d.reader.streamExact64(w, e.uncompressed_size); + }, + else => return Error.UnsupportedCompressionMethod, + } + } +}; + +/// The directory part of a zip entry name ("" at the top level). Zip names are +/// always '/'-separated, whatever the platform, so this is not path logic. +fn dirNameOf(name: []const u8) []const u8 { + const slash = std.mem.lastIndexOfScalar(u8, name, '/') orelse return ""; + return name[0..slash]; +} + +fn appendJsonString(gpa: Allocator, buf: *std.ArrayList(u8), s: []const u8) !void { + try buf.append(gpa, '"'); + for (s) |ch| switch (ch) { + '"' => try buf.appendSlice(gpa, "\\\""), + '\\' => try buf.appendSlice(gpa, "\\\\"), + '\n' => try buf.appendSlice(gpa, "\\n"), + '\r' => try buf.appendSlice(gpa, "\\r"), + '\t' => try buf.appendSlice(gpa, "\\t"), + else => if (ch < 0x20) { + try buf.print(gpa, "\\u{x:0>4}", .{ch}); + } else try buf.append(gpa, ch), + }; + try buf.append(gpa, '"'); +} + +// --------------------------------------------------------------------------- +// Tests. The fixtures are real zips, written here rather than committed: the +// reader's whole job is tolerating what a writer actually produces (a local +// header sized differently from the central one, a stored entry beside a +// deflated one), and a committed blob would freeze one writer's habits. +// --------------------------------------------------------------------------- + +const testing = std.testing; + +const TestFile = struct { name: []const u8, data: []const u8, deflate: bool }; + +/// Write a minimal but valid zip: local header + data per file, then the +/// central directory, then the end record. +fn writeTestZip(gpa: Allocator, io: std.Io, path: []const u8, files: []const TestFile) !void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + + var offsets = try gpa.alloc(u32, files.len); + defer gpa.free(offsets); + var packed_lens = try gpa.alloc(u32, files.len); + defer gpa.free(packed_lens); + var crcs = try gpa.alloc(u32, files.len); + defer gpa.free(crcs); + + for (files, 0..) |f, i| { + offsets[i] = @intCast(out.items.len); + crcs[i] = std.hash.Crc32.hash(f.data); + + var body: std.Io.Writer.Allocating = try .initCapacity(gpa, 4096); + defer body.deinit(); + if (f.deflate) { + var window: [flate.max_window_len]u8 = undefined; + var c = try flate.Compress.init(&body.writer, &window, .raw, .default); + try c.writer.writeAll(f.data); + try c.finish(); + } else { + try body.writer.writeAll(f.data); + } + const body_bytes = body.written(); + packed_lens[i] = @intCast(body_bytes.len); + + const lh: zip.LocalFileHeader = .{ + .signature = zip.local_file_header_sig, + .version_needed_to_extract = 20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = if (f.deflate) .deflate else .store, + .last_modification_time = 0, + .last_modification_date = 0, + .crc32 = crcs[i], + .compressed_size = packed_lens[i], + .uncompressed_size = @intCast(f.data.len), + .filename_len = @intCast(f.name.len), + .extra_len = 0, + }; + try out.appendSlice(gpa, std.mem.asBytes(&lh)); + try out.appendSlice(gpa, f.name); + try out.appendSlice(gpa, body_bytes); + } + + const cd_start: u32 = @intCast(out.items.len); + for (files, 0..) |f, i| { + const ch: zip.CentralDirectoryFileHeader = .{ + .signature = zip.central_file_header_sig, + .version_made_by = 20, + .version_needed_to_extract = 20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = if (f.deflate) .deflate else .store, + .last_modification_time = 0, + .last_modification_date = 0, + .crc32 = crcs[i], + .compressed_size = packed_lens[i], + .uncompressed_size = @intCast(f.data.len), + .filename_len = @intCast(f.name.len), + .extra_len = 0, + .comment_len = 0, + .disk_number = 0, + .internal_file_attributes = 0, + .external_file_attributes = 0, + .local_file_header_offset = offsets[i], + }; + try out.appendSlice(gpa, std.mem.asBytes(&ch)); + try out.appendSlice(gpa, f.name); + } + const cd_size: u32 = @intCast(out.items.len - cd_start); + + const end: zip.EndRecord = .{ + .signature = zip.end_record_sig, + .disk_number = 0, + .central_directory_disk_number = 0, + .record_count_disk = @intCast(files.len), + .record_count_total = @intCast(files.len), + .central_directory_size = cd_size, + .central_directory_offset = cd_start, + .comment_len = 0, + }; + try out.appendSlice(gpa, std.mem.asBytes(&end)); + + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = out.items }); +} + +/// tmpDir lives under the build root's .zig-cache; the reader takes paths +/// through cwd(), so the tests need its path rather than its handle. +fn tmpPath(gpa: Allocator, tmp: *std.testing.TmpDir) ![]u8 { + return std.fs.path.join(gpa, &.{ ".zig-cache", "tmp", &tmp.sub_path }); +} + +fn testIo() std.Io { + return std.Io.Threaded.global_single_threaded.io(); +} + +test "reads a stored and a deflated entry byte for byte" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // Long and repetitive, so deflate actually has something to do. + const big = "US5MD12M soundings " ** 400; + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/", .data = "", .deflate = false }, + .{ .name = "ENC_ROOT/A/A.000", .data = "stored cell bytes", .deflate = false }, + .{ .name = "ENC_ROOT/B/B.000", .data = big, .deflate = true }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The directory entry is not a chart and is not listed. + try testing.expectEqual(@as(usize, 2), arc.entries.len); + try testing.expectEqualStrings("ENC_ROOT/A/A.000", arc.entries[0].name); + + const a = try arc.readAlloc(gpa, io, 0, 1 << 20); + defer gpa.free(a); + try testing.expectEqualStrings("stored cell bytes", a); + + const b = try arc.readAlloc(gpa, io, arc.find("ENC_ROOT/B/B.000").?, 1 << 20); + defer gpa.free(b); + try testing.expectEqualStrings(big, b); + // Deflate really was exercised, not silently stored. + try testing.expect(arc.entries[1].compressed_size < arc.entries[1].uncompressed_size); +} + +test "an entry that claims more than the cap is refused before allocating" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "big.000", .data = "0123456789", .deflate = false }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + try testing.expectError(Error.EntryTooLarge, arc.readAlloc(gpa, io, 0, 4)); +} + +test "the update chain follows the cell and stops at the first gap" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // .003 is missing, so .004 is not part of the chain even though it is in + // the archive. A cell in another directory with the same stem must not be + // mistaken for an update of this one. + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = "base", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.001", .data = "u1", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.002", .data = "u2", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.004", .data = "u4", .deflate = false }, + .{ .name = "OTHER/US5MD12M/US5MD12M.003", .data = "elsewhere", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.TXT", .data = "notes", .deflate = false }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + const base = arc.find("ENC_ROOT/US5MD12M/US5MD12M.000").?; + const ups = try arc.updatesFor(gpa, base); + defer gpa.free(ups); + try testing.expectEqual(@as(usize, 2), ups.len); + try testing.expectEqualStrings("ENC_ROOT/US5MD12M/US5MD12M.001", arc.entries[ups[0]].name); + try testing.expectEqualStrings("ENC_ROOT/US5MD12M/US5MD12M.002", arc.entries[ups[1]].name); + + // A .TXT is not a base cell, so it has no chain. + const txt = arc.find("ENC_ROOT/US5MD12M/US5MD12M.TXT").?; + const none = try arc.updatesFor(gpa, txt); + defer gpa.free(none); + try testing.expectEqual(@as(usize, 0), none.len); +} + +test "a cell's directory names the files it references" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = "base", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/US348MDE.TXT", .data = "caution", .deflate = false }, + .{ .name = "ENC_ROOT/US5MD12M/PIC1.TIF", .data = "pic", .deflate = false }, + .{ .name = "ENC_ROOT/US4MD11M/US4MD11M.000", .data = "other", .deflate = false }, + .{ .name = "ENC_ROOT/US4MD11M/OTHER.TXT", .data = "not mine", .deflate = false }, + .{ .name = "CATALOG.031", .data = "catalogue", .deflate = false }, + }); + + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The cell's own directory, and nothing from the cell next door. + const base = arc.find("ENC_ROOT/US5MD12M/US5MD12M.000").?; + const sibs = arc.siblingsOf(base); + try testing.expectEqual(@as(usize, 3), sibs.len); + for (sibs) |si| { + try testing.expect(std.mem.startsWith(u8, arc.entries[si].name, "ENC_ROOT/US5MD12M/")); + } + + // An entry at the top level has no directory to share. + const cat = arc.find("CATALOG.031").?; + try testing.expectEqual(@as(usize, 1), arc.siblingsOf(cat).len); +} + +test "extractTo writes the entry under the caller's name" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + const body = "tiles and more tiles " ** 500; + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "../evil/USA.mbtiles", .data = body, .deflate = true }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + // The archive's own name is ignored: the destination is ours. + const out = try std.fs.path.join(gpa, &.{ dir, "safe.mbtiles" }); + defer gpa.free(out); + try arc.extractTo(io, 0, out); + + const got = try std.Io.Dir.cwd().readFileAlloc(io, out, gpa, .unlimited); + defer gpa.free(got); + try testing.expectEqualStrings(body, got); +} + +test "the listing is JSON and NUL-terminated" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + try writeTestZip(gpa, io, zpath, &.{ + .{ .name = "a\"quote\".000", .data = "xy", .deflate = false }, + }); + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + + const json = try arc.toJson(gpa); + defer gpa.free(json); + // The host reads this as a C string; without the terminator it reads on. + try testing.expectEqual(@as(u8, 0), json[json.len]); + try testing.expectEqualStrings("[{\"name\":\"a\\\"quote\\\".000\",\"size\":2,\"packed\":2}]", json); +} + +test "a file that is not a zip is refused at open" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "not.zip" }); + defer gpa.free(zpath); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = "0001 this is an S-57 cell, not an archive" }); + try testing.expectError(error.ZipNoEndRecord, Archive.open(gpa, io, zpath)); +} diff --git a/tools/bake.zig b/tools/bake.zig index bc6472b0..fa60513e 100644 --- a/tools/bake.zig +++ b/tools/bake.zig @@ -1,4 +1,4 @@ -//! `bake -o [--rules DIR] +//! `bake -o [--rules DIR] //! [-j N]` — produce a LIVE-composite structure on disk. Bake each chart to its OWN native-scale PMTiles under //! `//` (with its M_COVR coverage embedded in the metadata), then open a resident //! compositor over them and write the ownership partition to `/partition.tpart`. There is @@ -13,6 +13,7 @@ const compose = @import("compose"); // openComposeSourceFiles + serializePartiti const raster = @import("raster"); // the BSB/KAP bake + the raster chart handle const common = @import("common.zig"); const auxfiles = @import("engine").auxfiles; +const zipsrc = @import("engine").zipsrc; const Flags = common.Flags; const usageErr = common.usageErr; const resolveRulesDir = common.resolveRulesDir; @@ -168,34 +169,31 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { // The per-chart archive paths that back the compositor. var archive_paths = std.ArrayList([]const u8).empty; - var aux_written: usize = 0; { - // Bake each chart (dedup by stem — a boundary chart shared by two districts bakes once) - // to its own //.pmtiles, the shape an exchange set uses. + // Bake each chart to its own directory, the shape an exchange set uses. + // From a tree: dedup by stem (a boundary chart shared by two districts + // bakes once) into //.pmtiles. From an archive: + // mirror the entry's path, so what comes out is laid out like what + // went in. try std.Io.Dir.cwd().createDirPath(io, out_dir); var cell_paths = std.ArrayList([]const u8).empty; - var aux_files = std.ArrayList(auxfiles.File).empty; - if (std.mem.endsWith(u8, base_path, ".000")) { - try cell_paths.append(a, base_path); - // The cell's own directory holds the files it references. - if (want_aux) { - const stem = std.fs.path.stem(std.fs.path.basename(base_path)); - if (std.fs.path.dirname(base_path)) |cell_dir| { - var dir = std.Io.Dir.cwd().openDir(io, cell_dir, .{ .iterate = true }) catch null; - if (dir) |*d| { - defer d.close(io); - var it = d.iterate(); - while (it.next(io) catch null) |entry| { - if (entry.kind != .file or !auxfiles.isContent(entry.name)) continue; - const p = std.fs.path.join(a, &.{ cell_dir, entry.name }) catch continue; - const bytes = std.Io.Dir.cwd().readFileAlloc(io, p, a, .unlimited) catch continue; - aux_files.append(a, .{ .owner = stem, .name = entry.name, .bytes = bytes }) catch {}; - } - } - } + // A .zip is baked where it lies: the cells are named, not extracted, + // and each one is inflated as its turn comes. Nothing is unzipped, so + // importing NOAA's 788 MB All_ENCs.zip does not first cost 2 GiB of + // disk for source data that is dead as soon as the bake ends. + var zip_arc: ?*zipsrc.Archive = null; + if (std.mem.endsWith(u8, base_path, ".zip")) { + const arc = a.create(zipsrc.Archive) catch return usageErr("out of memory"); + arc.* = zipsrc.Archive.open(a, io, base_path) catch return usageErr("cannot read archive"); + zip_arc = arc; + for (arc.entries) |e| { + if (std.mem.endsWith(u8, e.name, ".000")) cell_paths.append(a, e.name) catch {}; } + std.debug.print("{s}: {d} entries, {d} cell(s)\n", .{ base_path, arc.entries.len, cell_paths.items.len }); + } else if (std.mem.endsWith(u8, base_path, ".000")) { + try cell_paths.append(a, base_path); } else { var dir = std.Io.Dir.cwd().openDir(io, base_path, .{ .iterate = true }) catch return usageErr("cannot open ENC_ROOT"); defer dir.close(io); @@ -207,16 +205,9 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { while (walker.next(io) catch null) |entry| { scan.step(); if (entry.kind != .file) continue; - if (want_aux and auxfiles.isContent(entry.path)) { - const p = std.fs.path.join(a, &.{ base_path, entry.path }) catch continue; - const bytes = std.Io.Dir.cwd().readFileAlloc(io, p, a, .unlimited) catch continue; - // The exchange set puts a cell's files in the cell's own - // directory, so that directory names the owner. Both names - // must be COPIED: the walker reuses one buffer for the path, - // so a borrowed slice becomes the next entry's name. - const owner = a.dupe(u8, std.fs.path.basename(std.fs.path.dirname(entry.path) orelse "")) catch continue; - const name = a.dupe(u8, entry.path) catch continue; - aux_files.append(a, .{ .owner = owner, .name = name, .bytes = bytes }) catch {}; + // The text and pictures a cell references are the engine's to + // write, beside the archive it bakes; the walk only counts them. + if (auxfiles.isContent(entry.path)) { scan.files += 1; continue; } @@ -238,28 +229,24 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { const stem = std.fs.path.stem(std.fs.path.basename(cp)); // One directory per chart, as the exchange set does it: the archive // and the files that chart references travel together. - const chart_dir = std.fs.path.join(a, &.{ out_dir, stem }) catch continue; + // + // From an archive the output MIRRORS the entry's own path, so + // ENC_ROOT/US5MD12M/US5MD12M.000 becomes + // /ENC_ROOT/US5MD12M/US5MD12M.pmtiles. The exchange set + // already gives each cell a directory, so mirroring lands the + // referenced text beside the right chart — and two districts that + // share a boundary cell keep their own copies instead of one + // overwriting the other. + const chart_dir = if (zip_arc != null) + std.fs.path.join(a, &.{ out_dir, std.fs.path.dirname(cp) orelse "" }) catch continue + else + std.fs.path.join(a, &.{ out_dir, stem }) catch continue; std.Io.Dir.cwd().createDirPath(io, chart_dir) catch continue; const name = std.fmt.allocPrint(a, "{s}.pmtiles", .{stem}) catch continue; out_paths.append(a, std.fs.path.join(a, &.{ chart_dir, name }) catch continue) catch {}; } if (out_paths.items.len != cell_paths.items.len) return usageErr("out of memory naming archives"); - // The referenced text and pictures, beside the chart that names them. - for (cell_paths.items) |cp| { - const stem = std.fs.path.stem(std.fs.path.basename(cp)); - var mine = std.ArrayList(auxfiles.File).empty; - for (aux_files.items) |af| { - if (std.mem.eql(u8, af.owner, stem)) mine.append(a, af) catch {}; - } - if (mine.items.len == 0) continue; - const chart_dir = std.fs.path.join(a, &.{ out_dir, stem }) catch continue; - aux_written += auxfiles.writeDir(io, a, chart_dir, mine.items) catch |err| blk: { - std.debug.print("warning: aux files not written for {s} ({s})\n", .{ stem, @errorName(err) }); - break :blk 0; - }; - } - const n_workers = @min(workers, cell_paths.items.len); if (cell_paths.items.len > 1) { std.debug.print("baking {d} cell(s) across {d} worker(s)…\n", .{ cell_paths.items.len, n_workers }); @@ -269,7 +256,10 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { .total = @intCast(cell_paths.items.len), .paths = cell_paths.items, }; - const baked = chart.bakeChartsToFiles(io, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell); + const baked = if (zip_arc) |arc| + chart.bakeZipChartsToFiles(io, arc, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell, want_aux) + else + chart.bakeChartsToFiles(io, cell_paths.items, out_paths.items, rules_dir, n_workers, null, &prog, onCell, want_aux); if (baked == 0) return usageErr("no cells baked (no .000 with M_COVR found)"); // bakeChartsToFiles reports a count, not which ones — a cell with no M_COVR @@ -306,10 +296,6 @@ pub fn run(io: std.Io, a: std.mem.Allocator, args: []const [:0]const u8) !void { const part_path = try std.fs.path.join(a, &.{ out_dir, "partition.tpart" }); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = part_path, .data = part_bytes }); - if (aux_written > 0) { - std.debug.print(" {d} auxiliary file(s) beside their charts\n", .{aux_written}); - } - std.debug.print( "live structure -> {s}/\n {d} per-chart directory(s) + partition.tpart (serve z {d}..{d})\n", .{ out_dir, src.readers.len, src.minz, src.loop_max }, diff --git a/tools/common.zig b/tools/common.zig index 408aaaf0..8c6e43a1 100644 --- a/tools/common.zig +++ b/tools/common.zig @@ -142,7 +142,7 @@ pub fn printUsage() void { \\{s} — offline S-57 -> PMTiles baker / inspector \\ \\usage: - \\ tile57 bake -o [--rules DIR] [-j N] [--no-aux] + \\ tile57 bake -o [--rules DIR] [-j N] [--no-aux] \\ Produce a live-composite structure: bake each chart (a single .000 + \\ its auto-discovered updates, OR every .000 in an ENC_ROOT, at \\ native band scale) to its own directory, //.pmtiles, diff --git a/tools/main.zig b/tools/main.zig index 1ca46565..f83f11ec 100644 --- a/tools/main.zig +++ b/tools/main.zig @@ -1,7 +1,7 @@ //! tile57 — the offline S-57 -> PMTiles baker / inspector CLI. //! //! Subcommands: -//! bake -o [--rules DIR] [-j N] +//! bake -o [--rules DIR] [-j N] //! Bake each chart to /tiles/.pmtiles and write the ownership //! partition to /partition.tpart — the live-composite structure a //! runtime compositor serves tiles from on demand. A BSB/KAP sheet bakes the