zip: bake charts straight out of an archive

build.zig

@@ -484,7 +484,16 @@ pub fn build(b: *std.Build) void {
 
     // All pure packages, imported by name into engine / libtile57.a / the baker.
     // (portray is libc, wired separately into the lib + baker only.)
+    // Charts read straight out of a .zip, and the text/pictures a cell points
+    // at. Both are pure std and both are needed by the engine root AND by the
+    // separately-compiled chart module, so they must be modules: a relative
+    // import from each would put one file in two modules.
+    const zipsrc_mod = b.addModule("zipsrc", .{ .root_source_file = b.path("src/zipsrc.zig") });
+    const auxfiles_mod = b.addModule("auxfiles", .{ .root_source_file = b.path("src/auxfiles.zig") });
+
     const pure_pkgs = [_]std.Build.Module.Import{
+        .{ .name = "zipsrc", .module = zipsrc_mod },
+        .{ .name = "auxfiles", .module = auxfiles_mod },
         .{ .name = "s57", .module = s57_mod },
         .{ .name = "s101", .module = s101_mod },
         .{ .name = "tiles", .module = tiles_mod },
@@ -681,6 +690,8 @@ pub fn build(b: *std.Build) void {
             .{ .name = "portray", .module = portray_mod },
             .{ .name = "sprite", .module = sprite_mod },
             .{ .name = "catalog", .module = catalog_embed },
+            .{ .name = "zipsrc", .module = zipsrc_mod },
+            .{ .name = "auxfiles", .module = auxfiles_mod },
         },
     });
     chart_mod.addImport("style", style_mod); // linestyle XML analysis
@@ -837,6 +848,10 @@ pub fn build(b: *std.Build) void {
     });
     _ = addPkgTest(b, test_step, "src/style/style.zig", target, optimize, &.{});
     _ = addPkgTest(b, test_step, "src/errors.zig", target, optimize, &.{});
+    // Charts read straight out of a .zip, and the aux files that travel with
+    // them: pure over std, so each tests alone.
+    _ = addPkgTest(b, test_step, "src/zipsrc.zig", target, optimize, &.{});
+    _ = addPkgTest(b, test_step, "src/auxfiles.zig", target, optimize, &.{});
     // Geometry core for the cross-band composition. No longer std-only: plane.zig
     // reads its partition tuning/stats valves via std.c.getenv, so the test binary
     // needs libc for the same reason compose's does, below.

include/tile57.h

@@ -290,6 +290,77 @@ tile57_status tile57_bake_rasters(const char *const *in_paths, const char *const
                                   tile57_bake_progress progress, tile57_bake_label label,
                                   void *progress_ctx, uint32_t *out_baked, tile57_error *err);
 
+/* ---- charts inside a .zip ---------------------------------------------------
+ *
+ * A chart archive arrives as one download and is mostly not the file you want:
+ * NOAA's All_ENCs.zip is 788 MB of deflate holding 2.0 GiB across 27,680
+ * entries. Unzipping it costs the mariner 2.0 GiB of disk that is dead the
+ * moment the import ends, on top of the charts they keep. These calls read the
+ * charts out of the archive where they lie: each cell is inflated when its turn
+ * comes, baked, and freed, so peak memory is the worker count times one chart
+ * and no intermediate copy is ever written.
+ *
+ * Every call opens the archive, works, and closes it. Walking the central
+ * directory of that 27,680-entry archive costs about 8 ms, so there is no
+ * handle to hold, close, or keep off another thread.
+ *
+ * `names` are entry names exactly as tile57_zip_list reports them. Destinations
+ * are always the CALLER's `out_paths`, never a name from the archive, so an
+ * entry called "../../etc/rc" cannot choose where anything lands. */
+
+/* List what a .zip holds, into *out / *out_len (free with tile57_free), as
+ * [{"name":..,"size":..,"packed":..}, ..] in central-directory order. Sizes are
+ * uncompressed and compressed bytes. Directory entries are omitted. The buffer
+ * is NUL-terminated past *out_len, so it reads as a C string too.
+ *
+ * This says what is IN the archive, not what is a chart: classification is the
+ * host's, and only the host knows which formats it supports. */
+tile57_status tile57_zip_list(const char *zip_path, uint8_t **out, size_t *out_len,
+                              tile57_error *err);
+
+/* tile57_bake_files, reading each cell straight out of `zip_path`: names[i] is
+ * a .000 entry, out_paths[i] is where its PMTiles goes. Its .001.. updates are
+ * found and applied from the archive, by the same rule the on-disk bake uses on
+ * a cell's directory: sequential from .001, stopping at the first gap.
+ *
+ * The text and pictures a cell references (TXTDSC, PICREP) are written beside
+ * the archive with an index.json manifest, so a pick report can resolve them
+ * through tile57_aux_open — but ONLY when out_paths[i] puts the archive in a
+ * directory named for the chart (<out>/US1EEZ3M/US1EEZ3M.pmtiles). Those files
+ * are named per exchange set rather than per chart, so charts baked flat into
+ * one directory would overwrite each other's manifests; rather than guess, the
+ * rule is the exchange set's own shape. Same contract otherwise — caller-owned
+ * lists, `label` names a finished chart by index, `progress` may cancel,
+ * *out_baked counts what was written. A name the archive does not hold is
+ * skipped, like a cell that fails to bake. */
+tile57_status tile57_bake_zip_charts(const char *zip_path, const char *const *names,
+                                     const char *const *out_paths, size_t n, uint32_t workers,
+                                     tile57_bake_progress progress, tile57_bake_label label,
+                                     void *progress_ctx, uint32_t *out_baked, tile57_error *err);
+
+/* tile57_bake_rasters, reading each BSB/KAP sheet straight out of `zip_path`.
+ * Same contract as tile57_bake_rasters, including that `workers` is a memory
+ * bound: a sheet is decoded whole whether it came from a file or an archive. */
+tile57_status tile57_bake_zip_rasters(const char *zip_path, const char *const *names,
+                                      const char *const *out_paths, size_t n, uint32_t workers,
+                                      tile57_bake_progress progress, tile57_bake_label label,
+                                      void *progress_ctx, uint32_t *out_baked, tile57_error *err);
+
+/* Inflate names[i] out of `zip_path` to out_paths[i], streaming: a 32 KiB
+ * window, whatever the file weighs. This is the path for what the engine can
+ * only read BY PATH — an .mbtiles or .pmtiles it memory-maps — where the file
+ * must exist on disk but a second copy of it need not. A 4 GiB .mbtiles
+ * therefore costs 4 GiB, not 8 GiB.
+ *
+ * Serial by design: these are the big entries, the disk is the limit, and
+ * parallel writers only contend for it. `progress` fires after each file and
+ * may cancel; *out_done counts what was written. An entry the archive does not
+ * hold is skipped. */
+tile57_status tile57_zip_extract(const char *zip_path, const char *const *names,
+                                 const char *const *out_paths, size_t n,
+                                 tile57_bake_progress progress, void *progress_ctx,
+                                 uint32_t *out_done, tile57_error *err);
+
 /* Read a PMTiles archive's metadata JSON blob (decompressed) into *out /
  * *out_len (free with tile57_free); NULL/0 when the archive carries none. A
  * per-chart bake embeds the chart's M_COVR coverage + cscl + date/name under a

src/bake_root.zig

@@ -29,6 +29,7 @@ pub const s101_adapter = root.s101_adapter;
 pub const catalogue = root.catalogue;
 pub const bake_enc = root.bake_enc;
 pub const auxfiles = root.auxfiles;
+pub const zipsrc = root.zipsrc; // charts read straight out of a .zip
 pub const geometry = @import("geometry"); // integer geometry: boolean, plane, partition
 
 pub const portray = @import("portray");

src/capi.zig

@@ -17,6 +17,7 @@ const mariner = @import("style").mariner;
 const style = @import("style");
 const errors = @import("errors"); // the engine error taxonomy + describe()
 const raster = @import("raster"); // raster charts (tile57_raster_chart_*)
+const zipsrc = @import("zipsrc"); // charts read straight out of a .zip
 // The S-52 ColorProfiles/colorProfile.xml baked into the library (build.zig), so
 // the style C ABI generates colortables + a base style template with no on-disk
 // catalogue. Symbols/linestyles are NOT embedded here (only the bake exe needs them).
@@ -412,7 +413,7 @@ export fn tile57_bake_files(
         in_list[i] = std.mem.span(ins[i]);
         out_list[i] = std.mem.span(outs[i]);
     }
-    const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label);
+    const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label, true);
     if (out_baked) |p| p.* = @intCast(baked);
     return OK;
 }
@@ -422,9 +423,12 @@ const RasterJob = struct {
     next: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
     done: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
     cancel: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
+    /// Sheet paths, or — when `zip` is set — names of entries inside it.
     in: []const []const u8,
     out: []const []const u8,
     ok: []bool,
+    /// The archive the sheets are read out of, or null to read from disk.
+    zip: ?*const zipsrc.Archive = null,
     progress: chart.BakeProgress,
     label: chart.BakeLabel,
     ctx: ?*anyopaque,
@@ -436,7 +440,14 @@ const RasterJob = struct {
 fn bakeOneRaster(io: std.Io, job: *RasterJob, i: usize) void {
     const a = std.heap.c_allocator;
     const stem = std.fs.path.stem(std.fs.path.basename(job.in[i]));
-    const kap = std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return;
+    // From the archive or from disk — the warp below cannot tell which.
+    const kap = blk: {
+        if (job.zip) |z| {
+            const idx = z.find(job.in[i]) orelse return;
+            break :blk z.readAlloc(a, io, idx, MAX_RASTER_BYTES) catch return;
+        }
+        break :blk std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return;
+    };
     defer a.free(kap);
     const baked = raster.bakebsb.bakeBytes(a, kap, stem) catch return;
     defer a.free(baked.bytes);
@@ -584,6 +595,174 @@ fn writeRasterPartition(out_list: []const []const u8, ok: []const bool) void {
     std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }) catch {};
 }
 
+// ---- charts inside a .zip ----------------------------------------------------
+// A chart archive is opened, read from, and closed within each call. Walking a
+// 27,680-entry central directory costs about 8 ms, so the alternative — an open
+// handle the host must hold, close, and keep off other threads — buys nothing
+// and can be got wrong.
+
+/// The largest a single raster sheet may claim to expand to. A KAP runs to tens
+/// of megabytes; the cap is here so a bad header fails instead of allocating.
+const MAX_RASTER_BYTES: u64 = 512 << 20;
+
+/// List what a .zip holds: [{"name":..,"size":..,"packed":..}, ..] into
+/// *out / *out_len (free with tile57_free). See tile57.h.
+export fn tile57_zip_list(zip_path: ?[*:0]const u8, out: ?*?[*]u8, out_len: ?*usize, err: ?*CError) callconv(.c) c_int {
+    const o, const n = bytesOut(out, out_len) catch return failWith(err, .badarg, bad_out);
+    const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null");
+    const io = sharedIo();
+    var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp);
+    defer arc.deinit();
+    const json = arc.toJson(gpa) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(json);
+    // Copied with its terminator: the payload is length-delimited by *out_len
+    // AND readable as a C string, so a host can take either.
+    const p = exportAlloc(json.len + 1) orelse return failWith(err, .nomem, "out of memory");
+    @memcpy(p[0 .. json.len + 1], json[0 .. json.len + 1]);
+    o.* = p;
+    n.* = json.len;
+    return OK;
+}
+
+/// Inflate entries out of a .zip to paths the CALLER names, streaming. See tile57.h.
+export fn tile57_zip_extract(
+    zip_path: ?[*:0]const u8,
+    names: ?[*]const [*:0]const u8,
+    out_paths: ?[*]const [*:0]const u8,
+    n: usize,
+    progress: chart.BakeProgress,
+    progress_ctx: ?*anyopaque,
+    out_done: ?*u32,
+    err: ?*CError,
+) callconv(.c) c_int {
+    if (out_done) |p| p.* = 0;
+    if (n == 0) return OK;
+    const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null");
+    const ns = names orelse return failWith(err, .badarg, "names must not be null");
+    const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null");
+
+    const io = sharedIo();
+    var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp);
+    defer arc.deinit();
+
+    // Serial on purpose: this is one file stream to disk per entry, and the
+    // entries that come this way are the big ones (a 4 GiB .mbtiles), where
+    // the disk is the limit and parallel writers only fight over it.
+    var done: u32 = 0;
+    for (0..n) |i| {
+        const name = std.mem.span(ns[i]);
+        const idx = arc.find(name) orelse continue;
+        arc.extractTo(io, idx, std.mem.span(outs[i])) catch continue;
+        done += 1;
+        if (progress) |cb| {
+            if (!cb(progress_ctx, @intCast(i + 1), @intCast(n))) break;
+        }
+    }
+    if (out_done) |p| p.* = done;
+    return OK;
+}
+
+/// tile57_bake_files reading the cells STRAIGHT OUT of a .zip. See tile57.h.
+export fn tile57_bake_zip_charts(
+    zip_path: ?[*:0]const u8,
+    names: ?[*]const [*:0]const u8,
+    out_paths: ?[*]const [*:0]const u8,
+    n: usize,
+    workers: u32,
+    progress: chart.BakeProgress,
+    label: chart.BakeLabel,
+    progress_ctx: ?*anyopaque,
+    out_baked: ?*u32,
+    err: ?*CError,
+) callconv(.c) c_int {
+    if (out_baked) |p| p.* = 0;
+    if (n == 0) return OK;
+    const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null");
+    const ns = names orelse return failWith(err, .badarg, "names must not be null");
+    const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null");
+
+    const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(in_list);
+    const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(out_list);
+    for (0..n) |i| {
+        in_list[i] = std.mem.span(ns[i]);
+        out_list[i] = std.mem.span(outs[i]);
+    }
+
+    const io = sharedIo();
+    var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp);
+    defer arc.deinit();
+    const baked = chart.bakeZipChartsToFiles(io, &arc, in_list, out_list, null, workers, progress, progress_ctx, label, true);
+    if (out_baked) |p| p.* = @intCast(baked);
+    return OK;
+}
+
+/// tile57_bake_rasters reading the sheets STRAIGHT OUT of a .zip. See tile57.h.
+export fn tile57_bake_zip_rasters(
+    zip_path: ?[*:0]const u8,
+    names: ?[*]const [*:0]const u8,
+    out_paths: ?[*]const [*:0]const u8,
+    n: usize,
+    workers: u32,
+    progress: chart.BakeProgress,
+    label: chart.BakeLabel,
+    progress_ctx: ?*anyopaque,
+    out_baked: ?*u32,
+    err: ?*CError,
+) callconv(.c) c_int {
+    if (out_baked) |p| p.* = 0;
+    if (n == 0) return OK;
+    const zp = spanOpt(zip_path) orelse return failWith(err, .badarg, "zip_path must not be null");
+    const ns = names orelse return failWith(err, .badarg, "names must not be null");
+    const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null");
+
+    const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(in_list);
+    const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(out_list);
+    const ok = gpa.alloc(bool, n) catch return failWith(err, .nomem, "out of memory");
+    defer gpa.free(ok);
+    @memset(ok, false);
+    for (0..n) |i| {
+        in_list[i] = std.mem.span(ns[i]);
+        out_list[i] = std.mem.span(outs[i]);
+    }
+
+    const io = sharedIo();
+    var arc = zipsrc.Archive.open(gpa, io, zp) catch |e| return failCtx(err, e, zp);
+    defer arc.deinit();
+
+    var job = RasterJob{
+        .in = in_list,
+        .out = out_list,
+        .ok = ok,
+        .zip = &arc,
+        .progress = progress,
+        .label = label,
+        .ctx = progress_ctx,
+    };
+    // Same deep stacks as tile57_bake_rasters: the warp is the same work,
+    // only the bytes arrive from the archive instead of a file.
+    const stack = 16 * 1024 * 1024;
+    var threads: [8]std.Thread = undefined;
+    const want = @min(@max(workers, 1), @min(threads.len, n));
+    var spawned: usize = 0;
+    while (spawned < want) : (spawned += 1) {
+        threads[spawned] = std.Thread.spawn(.{ .stack_size = stack }, rasterWorker, .{&job}) catch break;
+    }
+    if (spawned == 0) rasterWorker(&job);
+    for (threads[0..spawned]) |t| t.join();
+
+    var baked: u32 = 0;
+    for (ok) |o| {
+        if (o) baked += 1;
+    }
+    if (out_baked) |p| p.* = baked;
+    if (baked > 0) writeRasterPartition(out_list, ok);
+    return OK;
+}
+
 /// The metadata JSON blob of a PMTiles archive (decompressed) — e.g. the embedded
 /// per-cell "coverage" a single-cell bake carries — into *out / *out_len (free with
 /// tile57_free); NULL/0 when the archive carries none. See tile57.h.