From 921d2db2b34db75bda8a518d421d7e1b535f6ebe Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:36 +0100 Subject: [PATCH 01/18] poc for translating mirror urls --- src/main.zig | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 5bffbdf..926d590 100644 --- a/src/main.zig +++ b/src/main.zig @@ -436,13 +436,19 @@ pub fn main() !void { const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); + + const filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const mirror_url = try MirrorUrls.getUrl(arena, app_data_path, filename); + // log.info("{f}", .{std.json.fmt(mirror_urls.list.items, .{ .whitespace = .indent_4 })}); + const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - url.fetch, + mirror_url, .{ .debug_hash = false }, )); + log.info("downloaded {s} to '{}{s}'", .{ hashstore_name, global_cache_directory, hash.path() }); if (maybe_hash) |*previous_hash| { if (previous_hash.val.eql(&hash.val)) { @@ -873,6 +879,54 @@ fn makeOfficialUrl(arena: Allocator, semantic_version: SemanticVersion) Download }; } +pub const MirrorUrls = struct { + list: std.ArrayListUnmanaged([]const u8) = .empty, + pub const mirrorlist = struct { + pub const url = "https://ziglang.org/download/community-mirrors.txt"; + pub const uri = std.Uri.parse(url) catch unreachable; + }; + + fn getUrl( + arena: Allocator, + app_data_path: []const u8, + filename: []const u8, + ) ![]const u8 { + const self = try get(arena, app_data_path); + assert(self.list.items.len > 0); + return std.fmt.allocPrint(arena, "{s}{s}{s}?source=anyzig", .{ + self.list.items[0], + if (std.mem.endsWith(u8, self.list.items[0], "/")) "" else "/", + filename, + }); + } + + fn get( + arena: Allocator, + app_data_path: []const u8, + ) !@This() { + const mirrors_path = try std.fs.path.join(arena, &.{ app_data_path, "community-mirrors.txt" }); + var self: @This() = .{}; + + try fetchFile(arena, mirrorlist.url, mirrorlist.uri, mirrors_path); + + const mirrors_content = blk: { + // since we just downloaded the file, this should always succeed now + const file = try std.fs.cwd().openFile(mirrors_path, .{}); + defer file.close(); + break :blk try file.readToEndAlloc(arena, std.math.maxInt(usize)); + }; + var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); + while (iter.next()) |mirror| { + if (std.mem.startsWith(u8, mirror, "http")) { + try self.list.append(arena, mirror); + } + } + var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); + rand.random().shuffle([]const u8, self.list.items); + return self; + } +}; + fn getVersionUrl( arena: Allocator, app_data_path: []const u8, From 6a199d733908373c83a771b098be6f3aa6e84c1d Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:36 +0100 Subject: [PATCH 02/18] add minizign dependency --- build.zig | 7 ++++++- build.zig.zon | 4 ++++ src/main.zig | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 78105f4..3514b72 100644 --- a/build.zig +++ b/build.zig @@ -6,6 +6,8 @@ const Exe = enum { zig, zls }; pub fn build(b: *std.Build) !void { const zig_dep = b.dependency("zig", .{}); + const minizign_dep = b.dependency("minizign", .{}); + const minizign_mod = minizign_dep.module("minizign"); const version_option: ?[11]u8 = if (b.option( []const u8, @@ -47,6 +49,7 @@ pub fn build(b: *std.Build) !void { .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = dev_version_embed }, }, }), @@ -106,7 +109,7 @@ pub fn build(b: *std.Build) !void { ci_step.dependOn(test_step); ci_step.dependOn(&install_version_release_file.step); - try ci(b, &release_version, release_version_embed, zig_mod, ci_step, host_zip_exe); + try ci(b, &release_version, release_version_embed, zig_mod, minizign_mod, ci_step, host_zip_exe); } fn verifyForceVersion(v: []const u8) [11]u8 { @@ -593,6 +596,7 @@ fn ci( release_version: []const u8, release_version_embed: *std.Build.Module, zig_mod: *std.Build.Module, + minizign_mod: *std.Build.Module, ci_step: *std.Build.Step, host_zip_exe: *std.Build.Step.Compile, ) !void { @@ -633,6 +637,7 @@ fn ci( .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = release_version_embed }, }, }), diff --git a/build.zig.zon b/build.zig.zon index b26ede4..867c343 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -12,6 +12,10 @@ .url = "git+https://github.com/marler8997/zipcmdline#3dfca786a489d117e4b72ea10ffb4bbd9fc2dd72", .hash = "12201a08d7eff7619c8eb8284691a3ff959861b4bdd87216f180ed136672fb4ea26f", }, + .minizign = .{ + .url = "git+https://github.com/jedisct1/zig-minisign.git#50af1aa2ca0a4635e742b0a83735bccc3052b932", + .hash = "minizign-0.1.7-gg18cWlFAgBHo9NOoJiCzEn3dgmdE2bpWfQOuVipDhsk", + }, }, .paths = .{ "build.zig", diff --git a/src/main.zig b/src/main.zig index 926d590..e4b74eb 100644 --- a/src/main.zig +++ b/src/main.zig @@ -17,6 +17,7 @@ const Directory = std.Build.Cache.Directory; const EnvVar = std.zig.EnvVar; const zig = @import("zig"); +const minizign = @import("minizign"); const Package = zig.Package; const introspect = zig.introspect; From 27858a7e9abdd5da6e487eb6b97406284de9d193 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 03/18] wip on minisign validation --- build.zig.zon | 6 +-- src/main.zig | 105 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 80 insertions(+), 31 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 867c343..efedc83 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -2,11 +2,11 @@ .name = .anyzig, .version = "0.0.0", .fingerprint = 0x7cd092f5bd0fed33, // Changing this has security and trust implications. - .minimum_zig_version = "0.14.0", + .minimum_zig_version = "0.14.1", .dependencies = .{ .zig = .{ - .url = "git+https://github.com/ziglang/zig#5ad91a646a753cc3eecd8751e61cf458dadd9ac4", - .hash = "zig-0.0.0-Fp4XJDTgLgvQHn6QVvTe6INRdgdAzu43qE4JBXxhn9Ln", + .url = "git+https://github.com/ziglang/zig.git?ref=0.14.1#d03a147ea0a590ca711b3db07106effc559b0fc6", + .hash = "zig-0.0.0-Fp4XJConMAvbptegZi4D-tp5r9VKMY6KCB_mEm9mSzZN", }, .zip = .{ .url = "git+https://github.com/marler8997/zipcmdline#3dfca786a489d117e4b72ea10ffb4bbd9fc2dd72", diff --git a/src/main.zig b/src/main.zig index e4b74eb..3abea30 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,19 +434,29 @@ pub fn main() !void { else => |e| return e, } } + // TODO: cache mirror list + var mirrors = try MirrorUrls.get(gpa, app_data_path); + defer mirrors.deinit(gpa); + assert(mirrors.list.items.len > 0); const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const mirror_url = try MirrorUrls.getUrl(arena, app_data_path, filename); - // log.info("{f}", .{std.json.fmt(mirror_urls.list.items, .{ .whitespace = .indent_4 })}); + // TODO: retries + const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const mirror = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + defer mirror.deinit(gpa); + + defer std.fs.cwd().deleteFile(mirror.archive_path) catch {}; + try fetchFile(arena, mirror.archive_url, try std.Uri.parse(mirror.archive_url), mirror.archive_path); + + // TODO: download and verify minizign signature const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - mirror_url, + mirror.archive_path, .{ .debug_hash = false }, )); @@ -880,52 +890,91 @@ fn makeOfficialUrl(arena: Allocator, semantic_version: SemanticVersion) Download }; } -pub const MirrorUrls = struct { - list: std.ArrayListUnmanaged([]const u8) = .empty, - pub const mirrorlist = struct { - pub const url = "https://ziglang.org/download/community-mirrors.txt"; - pub const uri = std.Uri.parse(url) catch unreachable; - }; - - fn getUrl( - arena: Allocator, - app_data_path: []const u8, - filename: []const u8, - ) ![]const u8 { - const self = try get(arena, app_data_path); - assert(self.list.items.len > 0); - return std.fmt.allocPrint(arena, "{s}{s}{s}?source=anyzig", .{ - self.list.items[0], - if (std.mem.endsWith(u8, self.list.items[0], "/")) "" else "/", +const FetchInfo = struct { + archive_url: []const u8, + archive_path: []const u8, + minisign_url: []const u8, + minisign_path: []const u8, + fn init(gpa: Allocator, tmpdir: []const u8, mirror_url: []const u8, filename: []const u8) !@This() { + const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?source=anyzig", .{ + mirror_url, + if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", + filename, + }); + errdefer gpa.free(archive_url); + const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?source=anyzig", .{ + mirror_url, + if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, }); + errdefer gpa.free(minisign_url); + const minisign_filename = try std.mem.concat(gpa, u8, &.{ filename, ".minisig" }); + defer gpa.free(minisign_filename); + + const minisign_path = try std.fs.path.join(gpa, &.{ + tmpdir, + minisign_filename, + }); + return .{ + .archive_url = archive_url, + .minisign_url = minisign_url, + .minisign_path = minisign_path, + .archive_path = minisign_path[0 .. minisign_path.len - ".minisig".len], + }; + } + + fn deinit(self: @This(), gpa: Allocator) void { + gpa.free(self.archive_url); + gpa.free(self.minisign_url); + gpa.free(self.minisign_path); + // do not free archive_path here, since its a slice of minisign_path } +}; + +pub const MirrorUrls = struct { + list: std.ArrayListUnmanaged([]const u8) = .empty, + const mirrorlist = struct { + const url = "https://ziglang.org/download/community-mirrors.txt"; + const uri = std.Uri.parse(url) catch unreachable; + }; fn get( - arena: Allocator, - app_data_path: []const u8, + gpa: Allocator, + tmpdir: []const u8, ) !@This() { - const mirrors_path = try std.fs.path.join(arena, &.{ app_data_path, "community-mirrors.txt" }); + const mirrors_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); + defer gpa.free(mirrors_path); var self: @This() = .{}; - try fetchFile(arena, mirrorlist.url, mirrorlist.uri, mirrors_path); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + try fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrors_path); const mirrors_content = blk: { // since we just downloaded the file, this should always succeed now const file = try std.fs.cwd().openFile(mirrors_path, .{}); defer file.close(); - break :blk try file.readToEndAlloc(arena, std.math.maxInt(usize)); + break :blk try file.readToEndAlloc(gpa, std.math.maxInt(usize)); }; + defer gpa.free(mirrors_content); + var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); while (iter.next()) |mirror| { if (std.mem.startsWith(u8, mirror, "http")) { - try self.list.append(arena, mirror); + try self.list.append(gpa, try gpa.dupe(u8, mirror)); } } var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); rand.random().shuffle([]const u8, self.list.items); return self; } + + fn deinit(self: *@This(), gpa: Allocator) void { + for (self.list.items) |mirror| { + gpa.free(mirror); + } + self.list.deinit(gpa); + } }; fn getVersionUrl( @@ -1071,7 +1120,7 @@ fn fetchFile( const progress_node_name = std.fmt.allocPrint(scratch, "fetch {s}", .{uri}) catch |e| oom(e); defer scratch.free(progress_node_name); - const node = root.start(progress_node_name, 1); + const node = root.start(progress_node_name, 0); defer node.end(); const lock_filepath = try std.mem.concat(scratch, u8, &.{ out_filepath, ".lock" }); From 64e26f229d52c6ab0ef7d01072703ad341cd5fb0 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 04/18] add minisign validation --- src/main.zig | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 3abea30..2b28897 100644 --- a/src/main.zig +++ b/src/main.zig @@ -444,19 +444,21 @@ pub fn main() !void { // TODO: retries const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const mirror = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); - defer mirror.deinit(gpa); + const fetchinfo = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + defer fetchinfo.deinit(gpa); - defer std.fs.cwd().deleteFile(mirror.archive_path) catch {}; - try fetchFile(arena, mirror.archive_url, try std.Uri.parse(mirror.archive_url), mirror.archive_path); + defer std.fs.cwd().deleteFile(fetchinfo.archive_path) catch {}; + try fetchFile(arena, fetchinfo.archive_url, try std.Uri.parse(fetchinfo.archive_url), fetchinfo.archive_path); + defer std.fs.cwd().deleteFile(fetchinfo.minisign_path) catch {}; + try fetchFile(arena, fetchinfo.minisign_url, try std.Uri.parse(fetchinfo.minisign_url), fetchinfo.minisign_path); - // TODO: download and verify minizign signature + try fetchinfo.validateMinisign(gpa); const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - mirror.archive_path, + fetchinfo.archive_path, .{ .debug_hash = false }, )); @@ -923,6 +925,25 @@ const FetchInfo = struct { }; } + const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; + fn validateMinisign(self: @This(), gpa: Allocator) !void { + const sig_bytes = try std.fs.cwd().readFileAlloc(gpa, self.minisign_path, std.math.maxInt(u32)); + defer gpa.free(sig_bytes); + + const archive_file = try std.fs.cwd().openFile(self.archive_path, .{}); + defer archive_file.close(); + + var sig = try minizign.Signature.decode(gpa, sig_bytes); + defer sig.deinit(); + + try zig_org_minisign_pubkey.verifyFile( + gpa, + archive_file, + sig, + null, + ); + } + fn deinit(self: @This(), gpa: Allocator) void { gpa.free(self.archive_url); gpa.free(self.minisign_url); From 6538f1de49fbc90e263d25b1f1a292c85ded8b71 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 05/18] fix zls builds --- build.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/build.zig b/build.zig index 3514b72..5c16f90 100644 --- a/build.zig +++ b/build.zig @@ -655,6 +655,7 @@ fn ci( .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = release_version_embed }, }, }), From 3aa7631a7d1c90ae213a4ffe61f9bc6189637a58 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 06/18] implement mirror retries --- src/main.zig | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/main.zig b/src/main.zig index 2b28897..aa0cde6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -442,18 +442,22 @@ pub fn main() !void { const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - // TODO: retries const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const fetchinfo = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + var fetchinfo: FetchInfo = undefined; + var fetch_successful: bool = false; + for (mirrors.list.items) |mirror_url| { + fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); + errdefer fetchinfo.deinit(gpa); + fetchinfo.fetchAndValidate(gpa) catch continue; + fetch_successful = true; + break; + } + if (!fetch_successful) { + std.log.err("ran out of mirrors for: {s}", .{zig_archive_filename}); + return error.NoMoreMirrors; + } defer fetchinfo.deinit(gpa); - defer std.fs.cwd().deleteFile(fetchinfo.archive_path) catch {}; - try fetchFile(arena, fetchinfo.archive_url, try std.Uri.parse(fetchinfo.archive_url), fetchinfo.archive_path); - defer std.fs.cwd().deleteFile(fetchinfo.minisign_path) catch {}; - try fetchFile(arena, fetchinfo.minisign_url, try std.Uri.parse(fetchinfo.minisign_url), fetchinfo.minisign_path); - - try fetchinfo.validateMinisign(gpa); - const hash = hashAndPath(try cmdFetch( gpa, arena, @@ -910,7 +914,9 @@ const FetchInfo = struct { filename, }); errdefer gpa.free(minisign_url); - const minisign_filename = try std.mem.concat(gpa, u8, &.{ filename, ".minisig" }); + var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); + + const minisign_filename = try std.fmt.allocPrint(gpa, "tmp-{x}-{s}{s}", .{ rand.random().int(u32), filename, ".minisig" }); defer gpa.free(minisign_filename); const minisign_path = try std.fs.path.join(gpa, &.{ @@ -925,6 +931,25 @@ const FetchInfo = struct { }; } + fn deinit(self: @This(), gpa: Allocator) void { + std.fs.cwd().deleteFile(self.archive_path) catch {}; + std.fs.cwd().deleteFile(self.minisign_path) catch {}; + gpa.free(self.archive_url); + gpa.free(self.minisign_url); + gpa.free(self.minisign_path); + // do not free archive_path here, since its a slice of minisign_path + } + + fn fetchAndValidate(self: @This(), gpa: Allocator) !void { + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + + try fetchFile(arena.allocator(), self.archive_url, try std.Uri.parse(self.archive_url), self.archive_path); + try fetchFile(arena.allocator(), self.minisign_url, try std.Uri.parse(self.minisign_url), self.minisign_path); + + try self.validateMinisign(gpa); + } + const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; fn validateMinisign(self: @This(), gpa: Allocator) !void { const sig_bytes = try std.fs.cwd().readFileAlloc(gpa, self.minisign_path, std.math.maxInt(u32)); @@ -943,13 +968,6 @@ const FetchInfo = struct { null, ); } - - fn deinit(self: @This(), gpa: Allocator) void { - gpa.free(self.archive_url); - gpa.free(self.minisign_url); - gpa.free(self.minisign_path); - // do not free archive_path here, since its a slice of minisign_path - } }; pub const MirrorUrls = struct { From 9876488a237d1eccf296957ba1ee21b655352b98 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 07/18] better error messages - fetchFile still needs error handling --- src/main.zig | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main.zig b/src/main.zig index aa0cde6..709291c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -944,10 +944,29 @@ const FetchInfo = struct { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); - try fetchFile(arena.allocator(), self.archive_url, try std.Uri.parse(self.archive_url), self.archive_path); - try fetchFile(arena.allocator(), self.minisign_url, try std.Uri.parse(self.minisign_url), self.minisign_path); + fetchFile( + arena.allocator(), + self.archive_url, + try std.Uri.parse(self.archive_url), + self.archive_path, + ) catch |err| { + std.log.err("failed to download archive: {} {s}", .{ err, self.archive_url }); + return err; + }; + fetchFile( + arena.allocator(), + self.minisign_url, + try std.Uri.parse(self.minisign_url), + self.minisign_path, + ) catch |err| { + std.log.err("failed to download signature: {} {s}", .{ err, self.minisign_url }); + return err; + }; - try self.validateMinisign(gpa); + self.validateMinisign(gpa) catch |err| { + std.log.err("failed to validate: {} {s}", .{ err, self.archive_url }); + return err; + }; } const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; @@ -1327,7 +1346,7 @@ pub fn cmdFetch( }; defer fetch.deinit(); - log.info("downloading '{s}'...", .{url}); + log.info("cacheing '{s}'...", .{url}); fetch.run() catch |err| switch (err) { error.OutOfMemory => errExit("out of memory", .{}), error.FetchFailed => {}, // error bundle checked below From 39720666e7a169f6bf7b585db93aa18ce5d92d34 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 08/18] return more errors from fetchFile --- src/main.zig | 64 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/main.zig b/src/main.zig index 709291c..99e64b8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1190,39 +1190,53 @@ fn fetchFile( var client = std.http.Client{ .allocator = scratch }; defer client.deinit(); - client.initDefaultProxies(scratch) catch |err| std.debug.panic( - "fetch '{}': init proxy failed with {s}", - .{ uri, @errorName(err) }, - ); + client.initDefaultProxies(scratch) catch |err| { + log.err( + "fetch '{}': init proxy failed with {s}", + .{ uri, @errorName(err) }, + ); + return err; + }; + var header_buffer: [4096]u8 = undefined; - var request = client.open(.GET, uri, .{ + var request = try client.open(.GET, uri, .{ .server_header_buffer = &header_buffer, .keep_alive = false, - }) catch |e| std.debug.panic( - "fetch '{}': connect failed with {s}", - .{ uri, @errorName(e) }, - ); + }); defer request.deinit(); - request.send() catch |e| std.debug.panic( - "fetch '{}': send failed with {s}", - .{ uri, @errorName(e) }, - ); - request.wait() catch |e| std.debug.panic( - "fetch '{}': wait failed with {s}", - .{ uri, @errorName(e) }, - ); - if (request.response.status != .ok) return errExit( - "fetch '{}': HTTP response {} \"{?s}\"", - .{ uri, @intFromEnum(request.response.status), request.response.status.phrase() }, - ); + request.send() catch |e| { + log.err( + "fetch '{}': send failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; + request.wait() catch |e| { + std.debug.panic( + "fetch '{}': wait failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; + + if (request.response.status != .ok) { + log.err( + "fetch '{}': HTTP response {} \"{?s}\"", + .{ uri, @intFromEnum(request.response.status), request.response.status.phrase() }, + ); + return error.BadHttpStatus; + } const out_filepath_tmp = std.mem.concat(scratch, u8, &.{ out_filepath, ".fetching" }) catch |e| oom(e); defer scratch.free(out_filepath_tmp); - const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| std.debug.panic( - "create '{s}' failed with {s}", - .{ out_filepath_tmp, @errorName(e) }, - ); + const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| { + std.log.err( + "create '{s}' failed with {s}", + .{ out_filepath_tmp, @errorName(e) }, + ); + return error.CouldNotCreateFile; + }; defer { if (std.fs.cwd().deleteFile(out_filepath_tmp)) { std.log.info("removed '{s}'", .{out_filepath_tmp}); From 654376ea8f563045760ef959d3af6720214c6c43 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 09/18] fix memory leak in FetchInfo error cases --- src/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 99e64b8..1568081 100644 --- a/src/main.zig +++ b/src/main.zig @@ -447,8 +447,10 @@ pub fn main() !void { var fetch_successful: bool = false; for (mirrors.list.items) |mirror_url| { fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); - errdefer fetchinfo.deinit(gpa); - fetchinfo.fetchAndValidate(gpa) catch continue; + fetchinfo.fetchAndValidate(gpa) catch { + fetchinfo.deinit(gpa); + continue; + }; fetch_successful = true; break; } From 815a3fbbcaaa38fd5c4286a445f5475f7bcbf787 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 10/18] move mirror loop out of main --- src/main.zig | 53 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/main.zig b/src/main.zig index 1568081..1cde6a2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -443,21 +443,8 @@ pub fn main() !void { defer url.deinit(arena); const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - var fetchinfo: FetchInfo = undefined; - var fetch_successful: bool = false; - for (mirrors.list.items) |mirror_url| { - fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); - fetchinfo.fetchAndValidate(gpa) catch { - fetchinfo.deinit(gpa); - continue; - }; - fetch_successful = true; - break; - } - if (!fetch_successful) { - std.log.err("ran out of mirrors for: {s}", .{zig_archive_filename}); - return error.NoMoreMirrors; - } + + const fetchinfo = try mirrors.fetchFromAny(gpa, app_data_path, zig_archive_filename); defer fetchinfo.deinit(gpa); const hash = hashAndPath(try cmdFetch( @@ -903,27 +890,29 @@ const FetchInfo = struct { archive_path: []const u8, minisign_url: []const u8, minisign_path: []const u8, + const anyzig_mirror_url_query = "source=anyzig/" ++ @embedFile("version"); fn init(gpa: Allocator, tmpdir: []const u8, mirror_url: []const u8, filename: []const u8) !@This() { - const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?source=anyzig", .{ + const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?{s}", .{ mirror_url, if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, + anyzig_mirror_url_query, }); errdefer gpa.free(archive_url); - const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?source=anyzig", .{ + const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?{s}", .{ mirror_url, if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, + anyzig_mirror_url_query, }); errdefer gpa.free(minisign_url); - var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); - const minisign_filename = try std.fmt.allocPrint(gpa, "tmp-{x}-{s}{s}", .{ rand.random().int(u32), filename, ".minisig" }); - defer gpa.free(minisign_filename); + const minisig_filename = try std.fmt.allocPrint(gpa, "tmp-{s}{s}", .{ filename, ".minisig" }); + defer gpa.free(minisig_filename); const minisign_path = try std.fs.path.join(gpa, &.{ tmpdir, - minisign_filename, + minisig_filename, }); return .{ .archive_url = archive_url, @@ -1020,7 +1009,8 @@ pub const MirrorUrls = struct { var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); while (iter.next()) |mirror| { - if (std.mem.startsWith(u8, mirror, "http")) { + const mirror_without_whitespace = std.mem.trim(u8, mirror, &std.ascii.whitespace); + if (mirror_without_whitespace.len > 0) { try self.list.append(gpa, try gpa.dupe(u8, mirror)); } } @@ -1029,6 +1019,25 @@ pub const MirrorUrls = struct { return self; } + fn fetchFromAny(self: @This(), gpa: Allocator, tmpdir: []const u8, filename: []const u8) !FetchInfo { + var fetchinfo: FetchInfo = undefined; + var fetch_successful: bool = false; + for (self.list.items) |mirror_url| { + fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); + fetchinfo.fetchAndValidate(gpa) catch { + fetchinfo.deinit(gpa); + continue; + }; + fetch_successful = true; + break; + } + if (!fetch_successful) { + log.err("ran out of mirrors for: {s}", .{filename}); + return error.NoMoreMirrors; + } + return fetchinfo; + } + fn deinit(self: *@This(), gpa: Allocator) void { for (self.list.items) |mirror| { gpa.free(mirror); From 26384b1387383db2ca3a4e28f55d76c96de6d909 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 11/18] add log messages when deleting tempfile fails --- src/main.zig | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 1cde6a2..06d6dbe 100644 --- a/src/main.zig +++ b/src/main.zig @@ -923,8 +923,14 @@ const FetchInfo = struct { } fn deinit(self: @This(), gpa: Allocator) void { - std.fs.cwd().deleteFile(self.archive_path) catch {}; - std.fs.cwd().deleteFile(self.minisign_path) catch {}; + std.fs.cwd().deleteFile(self.archive_path) catch |err| switch (err) { + error.FileNotFound => {}, + else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + }; + std.fs.cwd().deleteFile(self.minisign_path) catch |err| switch (err) { + error.FileNotFound => {}, + else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + }; gpa.free(self.archive_url); gpa.free(self.minisign_url); gpa.free(self.minisign_path); From 3b29a17a6eab9b2ca374e99249d88ae3162d2f9d Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 12/18] remove panics from fetchFile --- src/main.zig | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/main.zig b/src/main.zig index 06d6dbe..7291a6a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1026,22 +1026,16 @@ pub const MirrorUrls = struct { } fn fetchFromAny(self: @This(), gpa: Allocator, tmpdir: []const u8, filename: []const u8) !FetchInfo { - var fetchinfo: FetchInfo = undefined; - var fetch_successful: bool = false; for (self.list.items) |mirror_url| { - fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); + const fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); fetchinfo.fetchAndValidate(gpa) catch { fetchinfo.deinit(gpa); continue; }; - fetch_successful = true; - break; - } - if (!fetch_successful) { - log.err("ran out of mirrors for: {s}", .{filename}); - return error.NoMoreMirrors; + return fetchinfo; } - return fetchinfo; + log.err("ran out of mirrors for: {s}", .{filename}); + return error.NoMoreMirrors; } fn deinit(self: *@This(), gpa: Allocator) void { @@ -1229,7 +1223,7 @@ fn fetchFile( return e; }; request.wait() catch |e| { - std.debug.panic( + log.err( "fetch '{}': wait failed with {s}", .{ uri, @errorName(e) }, ); @@ -1248,7 +1242,7 @@ fn fetchFile( defer scratch.free(out_filepath_tmp); const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| { - std.log.err( + log.err( "create '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }, ); @@ -1256,10 +1250,10 @@ fn fetchFile( }; defer { if (std.fs.cwd().deleteFile(out_filepath_tmp)) { - std.log.info("removed '{s}'", .{out_filepath_tmp}); + log.info("removed '{s}'", .{out_filepath_tmp}); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| std.log.err("remove '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }), + else => |e| log.err("remove '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }), } file.close(); } @@ -1269,7 +1263,7 @@ fn fetchFile( // not sure if it's a problem with the mach server or Zig's HTTP client if (request.response.content_length) |content_length| { if (std.mem.eql(u8, url_string, DownloadIndexKind.mach.url())) { - std.log.warn("ignoring content length {} for mach index", .{content_length}); + log.warn("ignoring content length {} for mach index", .{content_length}); break :blk null; } } @@ -1291,10 +1285,13 @@ fn fetchFile( total_received += len; if (maybe_content_length) |content_length| { - if (total_received > content_length) errExit( - "fetch '{}': read more than Content-Length ({})", - .{ uri, content_length }, - ); + if (total_received > content_length) { + log.err( + "fetch '{}': read more than Content-Length ({})", + .{ uri, content_length }, + ); + return error.ContentLengthMismatch; + } } // NOTE: not going through a buffered writer since we're writing // large chunks From 006daeab2f8e7aeff50468624aa93661c84cca5a Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:37 +0100 Subject: [PATCH 13/18] remove more panics from fetchFile --- src/main.zig | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main.zig b/src/main.zig index 7291a6a..4d53470 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1277,10 +1277,13 @@ fn fetchFile( var total_received: u64 = 0; while (true) { var buf: [@max(std.heap.page_size_min, 4096)]u8 = undefined; - const len = request.reader().read(&buf) catch |e| std.debug.panic( - "fetch '{}': read failed with {s}", - .{ uri, @errorName(e) }, - ); + const len = request.reader().read(&buf) catch |e| { + log.err( + "fetch '{}': read failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; if (len == 0) break; total_received += len; @@ -1295,17 +1298,23 @@ fn fetchFile( } // NOTE: not going through a buffered writer since we're writing // large chunks - file.writer().writeAll(buf[0..len]) catch |err| std.debug.panic( - "fetch '{}': write {} bytes of HTTP response failed with {s}", - .{ uri, len, @errorName(err) }, - ); + file.writer().writeAll(buf[0..len]) catch |err| { + log.err( + "fetch '{}': write {} bytes of HTTP response failed with {s}", + .{ uri, len, @errorName(err) }, + ); + return err; + }; } if (maybe_content_length) |content_length| { - if (total_received != content_length) errExit( - "fetch '{}': Content-Length is {} but only read {}", - .{ uri, content_length, total_received }, - ); + if (total_received != content_length) { + log.err( + "fetch '{}': Content-Length is {} but only read {}", + .{ uri, content_length, total_received }, + ); + return error.ContentLengthMismatch; + } } try std.fs.cwd().rename(out_filepath_tmp, out_filepath); From 8d29213a2602add89f60de089cbdc67a4672ae60 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:38 +0100 Subject: [PATCH 14/18] catch errors on mirrorlist download, so we can reuse an existing one --- src/main.zig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 4d53470..405b522 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,7 +434,6 @@ pub fn main() !void { else => |e| return e, } } - // TODO: cache mirror list var mirrors = try MirrorUrls.get(gpa, app_data_path); defer mirrors.deinit(gpa); assert(mirrors.list.items.len > 0); @@ -997,17 +996,19 @@ pub const MirrorUrls = struct { gpa: Allocator, tmpdir: []const u8, ) !@This() { - const mirrors_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); - defer gpa.free(mirrors_path); + const mirrorlist_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); + defer gpa.free(mirrorlist_path); var self: @This() = .{}; var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); - try fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrors_path); + fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrorlist_path) catch { + log.err("failed to fetch mirrorlist to: {s}", .{mirrorlist_path}); + }; const mirrors_content = blk: { - // since we just downloaded the file, this should always succeed now - const file = try std.fs.cwd().openFile(mirrors_path, .{}); + // load the mirrorlist we just downloaded, or is still around from a previous run + const file = try std.fs.cwd().openFile(mirrorlist_path, .{}); defer file.close(); break :blk try file.readToEndAlloc(gpa, std.math.maxInt(usize)); }; From 0ed91689e861b46273a2442a22dc835788ac0c98 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:38 +0100 Subject: [PATCH 15/18] put downloads in a subdirectory --- src/main.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.zig b/src/main.zig index 405b522..4c5e5a3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -443,7 +443,8 @@ pub fn main() !void { const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const fetchinfo = try mirrors.fetchFromAny(gpa, app_data_path, zig_archive_filename); + const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); + const fetchinfo = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); defer fetchinfo.deinit(gpa); const hash = hashAndPath(try cmdFetch( @@ -906,7 +907,7 @@ const FetchInfo = struct { }); errdefer gpa.free(minisign_url); - const minisig_filename = try std.fmt.allocPrint(gpa, "tmp-{s}{s}", .{ filename, ".minisig" }); + const minisig_filename = try std.fmt.allocPrint(gpa, "{s}{s}", .{ filename, ".minisig" }); defer gpa.free(minisig_filename); const minisign_path = try std.fs.path.join(gpa, &.{ @@ -928,7 +929,7 @@ const FetchInfo = struct { }; std.fs.cwd().deleteFile(self.minisign_path) catch |err| switch (err) { error.FileNotFound => {}, - else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + else => std.log.err("remove '{s}' failed with {s}", .{ self.minisign_path, @errorName(err) }), }; gpa.free(self.archive_url); gpa.free(self.minisign_url); From 0e29fd7f4e40905ca99e3cc56e9be60088b19285 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:38 +0100 Subject: [PATCH 16/18] only use mirrors on downloads from ziglang.org --- src/main.zig | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main.zig b/src/main.zig index 4c5e5a3..77736f9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,24 +434,31 @@ pub fn main() !void { else => |e| return e, } } - var mirrors = try MirrorUrls.get(gpa, app_data_path); - defer mirrors.deinit(gpa); - assert(mirrors.list.items.len > 0); - const url = try getVersionUrl(arena, app_data_path, semantic_version); + var url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const fetchinfo: ?FetchInfo = if (std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) fetchinfo: { + var mirrors = try MirrorUrls.get(gpa, app_data_path); + defer mirrors.deinit(gpa); + assert(mirrors.list.items.len > 0); - const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); - const fetchinfo = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); - defer fetchinfo.deinit(gpa); + const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + + const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); + const fi = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); + url.fetch = fi.archive_path; + break :fetchinfo fi; + } else null; + defer { + if (fetchinfo) |fi| fi.deinit(gpa); + } const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - fetchinfo.archive_path, + url.fetch, .{ .debug_hash = false }, )); From f0aa14ac72c10adf624f985e5a60b39083fe1c59 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:38 +0100 Subject: [PATCH 17/18] handle error on fetching mirrorlist --- src/main.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main.zig b/src/main.zig index 77736f9..0142854 100644 --- a/src/main.zig +++ b/src/main.zig @@ -438,10 +438,15 @@ pub fn main() !void { var url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const fetchinfo: ?FetchInfo = if (std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) fetchinfo: { + const fetchinfo: ?FetchInfo = if (!std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) + null + else fetchinfo: { var mirrors = try MirrorUrls.get(gpa, app_data_path); defer mirrors.deinit(gpa); - assert(mirrors.list.items.len > 0); + if (mirrors.list.items.len == 0) { + log.err("no zig mirrors found.", .{}); + break :fetchinfo null; + } const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; @@ -449,7 +454,8 @@ pub fn main() !void { const fi = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); url.fetch = fi.archive_path; break :fetchinfo fi; - } else null; + }; + defer { if (fetchinfo) |fi| fi.deinit(gpa); } From bbdb32e4e66a24574666a3a188a4c1dea6b45e83 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 16 Feb 2026 18:32:38 +0100 Subject: [PATCH 18/18] remove deinit on arena allocated urls --- src/main.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 0142854..5065da0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -436,7 +436,6 @@ pub fn main() !void { } var url = try getVersionUrl(arena, app_data_path, semantic_version); - defer url.deinit(arena); const fetchinfo: ?FetchInfo = if (!std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) null