diff --git a/.vscode/cspell/zig.cspell b/.vscode/cspell/zig.cspell index 466d704..e5859b8 100644 --- a/.vscode/cspell/zig.cspell +++ b/.vscode/cspell/zig.cspell @@ -1,18 +1,16 @@ -comptime -struct -nosuspend -usingnamespace +anyerror +anyopaque +anytype +bitstack callconv -orelse +comptime +deinit errdefer - -usize isize -anytype -anyopaque -anyerror - -deinit - memcpy -memset \ No newline at end of file +memset +nosuspend +orelse +struct +usingnamespace +usize diff --git a/.vscode/settings.json b/.vscode/settings.json index 453bb17..58329e0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,5 @@ "Minifloat", "signedness" ], - "zig.zls.enableBuildOnSave": true + "zig.buildOnSaveProvider": "zls" } \ No newline at end of file diff --git a/README.md b/README.md index af72c38..5769a23 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,16 @@ Harmony Binary Protocol (HBP) is a general purpose serialization protocol inspired by the protocols like Bolt's PackStream and Redis's RESP3, that aims to provide a standardized type aware way to serialize and deserialize data. -It consists of a set of basic types, composite types, and meta types that together allow you to represent data in the way you need. +It consists of a set of primitive types, composite types, and meta types that together allow you to represent data in the way you need. # Table of Contents + - [About](#about) - [Table of Contents](#table-of-contents) - [Representation](#representation) -- [Basic Data Types](#basic-data-types) +- [Identifier](#identifier) +- [Primitive Data Types](#primitive-data-types) - [Null](#null) - [Bool](#bool) - [Numbers](#numbers) @@ -21,27 +23,33 @@ It consists of a set of basic types, composite types, and meta types that togeth - [Floats](#floats) - [Decimals](#decimals) - [Meta Data Types](#meta-data-types) - - [String](#string) - - [Vector](#vector) - [Optional](#optional) - [Enum](#enum) + - [Union](#union) - [Error](#error) - [Composite Data Types](#composite-data-types) - - [Array](#array) - - [List](#list) + - [Strings](#strings) + - [Tuple](#tuple) + - [Vector](#vector) - [Dictionary](#dictionary) - [Map](#map) -- [Cheat Sheet](#cheat-sheet) + # Representation Every serialized HBP value begins with the HBP version used to encode it followed by a marker that represents the type of the data. -![representation image](representation.png) +# Identifier + +The HBP identifier is the byte at the beginning of every payload that tells you the version of the protocol and allows developers to pass custom flags. -# Basic Data Types +The identifier is a single byte subdivided into 2 parts of 4 bits. +The first 4 bits are reserved for the protocol version, parsers use this information to adapt to version-specific changes. +The last 4 bits are used for user defined flags, spec compliant parsers will not validate or make use of this bits for anything, they are given to the user as-is. -Basic types (or primitives) are the fundamental blocks used to represent the encoded data. +# Primitive Data Types + +Primitive types (or primitives) are the fundamental blocks used to represent the encoded data. ## Null @@ -101,6 +109,9 @@ Arbitrary sized integers are followed by `2` bytes designating their bit-width, ### Floats +> [!CAUTION] +> Not all float formats are implemented yet, they are present on the spec for future proofing. + | Marker | Data Size (bytes) | Type | | :----: | :---------------: | :-----------------------------------------------------------------------------------------------------------: | | `30` | 2 | [IEEE 754 Half precision float](https://en.wikipedia.org/wiki/Half-precision_floating-point_format) | @@ -115,6 +126,9 @@ Arbitrary sized integers are followed by `2` bytes designating their bit-width, ### Decimals +> [!CAUTION] +> Decimals are not yet supported, progress can be tracked [here](https://github.com/ziglang/zig/issues/4221). + | Marker | Data Size (bytes) | Type | | :----: | :---------------: | :-----------------------------------------------------------------------------------: | | `3A` | 4 | [IEEE 754 Decimal32](https://en.wikipedia.org/wiki/Decimal32_floating-point_format) | @@ -127,39 +141,29 @@ Meta data types are special types that acts as metadata for other types and they HBP reserves all the `E0-FF` range for meta types. -## String - -Marker: `E0` - -A String is a character encoded list of bytes. - -The string marker is always followed by a list marker with the type byte set to one of the following: - -| Byte | Encoding | -| ---- | -------- | -| `20` | UTF-8 | -| `21` | UTF-16 | - -## Vector - -Marker: `E3` - -The vector marker must **always** be followed by a list marker to indicate the size and type of the vector, the data should follow the same encoding as the indicated type. - ## Optional -Marker: `F0` +Marker: `E0` This marker is used as an indicator that the following marker can either be [`null`](#null) or another type. ## Enum -Marker: `F1` +Marker: `E1` The enum marker must **always** be followed by an integer marker to indicate the maximum size of the enum, the data should follow the same encoding as the indicated type. +## Union + +Marker: `E2` + +The union marker is **always** followed by an integer marker indicating the active tag followed by the union data. + ## Error +> [!WARNING] +> Error meta types are not yet supported by the zig implementation + Marker: `FF` Example: @@ -172,11 +176,47 @@ Serialized: 01 FF 6B 54 68 69 73 20 46 61 69 6C 65 64 # Composite Data Types -## Array +## Strings + +Strings are `UTF-8` encoded arrays of bytes. + +> Why aren't strings a meta type on top of Vector? There was a long discussion about this that needs to be appended here. . . -Small arrays: +Small strings: + +| Marker | String size | +| :----: | :---------: | +| `60` | 0 | +| `61` | 1 | +| `62` | 2 | +| `63` | 3 | +| `64` | 4 | +| `65` | 5 | +| `66` | 6 | +| `67` | 7 | +| `68` | 8 | +| `69` | 9 | +| `6A` | 10 | +| `6B` | 11 | +| `6C` | 12 | +| `6D` | 13 | +| `6E` | 14 | +| `6F` | 15 | + +Long Strings: + +| Marker | Extra bytes | Maximum Size | +| :----: | :---------: | :-----------: | +| `C0` | 1 | 255 | +| `C1` | 2 | 65_535 | +| `C2` | 4 | 4_294_967_295 | -| Marker | Array size | + +## Tuple + +Small tuples: + +| Marker | Tuple size | | :----: | :--------: | | `70` | 0 | | `71` | 1 | @@ -195,7 +235,7 @@ Small arrays: | `7E` | 14 | | `7F` | 15 | -Long arrays: +Long tuples: | Marker | Extra bytes | Maximum Size | | :----: | :---------: | :-----------: | @@ -203,7 +243,7 @@ Long arrays: | `DB` | 2 | 65_535 | | `DC` | 4 | 4_294_967_295 | -An array is a list of values, each one serializing their own type alongside like a basic hbp payload. If its a long array, the length will come **after** the value type. +A tuple is a list of values, each one serializing their own type alongside like a basic hbp payload. If its a long array, the length will come **after** the value type. ```txt Original: [3, 6, 9] @@ -212,30 +252,30 @@ Serialized: 01 73 10 03 10 06 10 09 ``` -## List - -Small lists: - -| Marker | List Size | -| :----: | :-------: | -| `80` | 0 | -| `81` | 1 | -| `82` | 2 | -| `83` | 3 | -| `84` | 4 | -| `85` | 5 | -| `86` | 6 | -| `87` | 7 | -| `88` | 8 | -| `89` | 9 | -| `8A` | 10 | -| `8B` | 11 | -| `8C` | 12 | -| `8D` | 13 | -| `8E` | 14 | -| `8F` | 15 | +## Vector -Long lists: +Small vectors: + +| Marker | Vector Size | +| :----: | :---------: | +| `80` | 0 | +| `81` | 1 | +| `82` | 2 | +| `83` | 3 | +| `84` | 4 | +| `85` | 5 | +| `86` | 6 | +| `87` | 7 | +| `88` | 8 | +| `89` | 9 | +| `8A` | 10 | +| `8B` | 11 | +| `8C` | 12 | +| `8D` | 13 | +| `8E` | 14 | +| `8F` | 15 | + +Long vectors: | Marker | Extra bytes | Maximum Size | | :----: | :---------: | :-----------: | @@ -243,12 +283,12 @@ Long lists: | `DE` | 2 | 65_535 | | `DF` | 4 | 4_294_967_295 | -A list as the name indicates is a list of values where all the values have the same type which has to be indicated right after the list marker. If its a long list, the length will come **after** the value type. +A vector is a known-type list of items. The vector marker is followed by [primitive data type](#primitive-data-types) and all elements will follow the encoding of that type. ```txt -Original: List([3, 6, 9]) +Original: Vector(u8, [3, 6, 9]) -Serialized: 01 83 10 03 06 09 +Serialized: 01 83 20 03 06 09 ``` ## Dictionary @@ -272,6 +312,9 @@ The encoding of a dictionary is as follows: ## Map +> [!CAUTION] +> Maps are not yet implemented. + | Marker | Extra bytes | Maximum Size | | :----: | :---------: | :-----------: | | `D3` | 1 | 255 | @@ -279,29 +322,3 @@ The encoding of a dictionary is as follows: | `D5` | 4 | 4_294_967_295 | A map is just like a dictionary but instead, the keys can be of any type. - -# Cheat Sheet - -| Marker | Name | Type | -| :-----: | :--------------------------------------: | :--------------------------------: | -| `00` | [`null`](#null) | [Primitive](#basic-data-types) | -| `01` | [`false`](#bool) | [Primitive](#basic-data-types) | -| `02` | [`true`](#bool) | [Primitive](#basic-data-types) | -| `10-16` | [`signed integer`](#signed-integers) | [Primitive](#basic-data-types) | -| `1F` | [`signed integer`](#signed-integers) | [Primitive](#basic-data-types) | -| `20-26` | [`unsigned integer`](#unsigned-integers) | [Primitive](#basic-data-types) | -| `2F` | [`unsigned integer`](#unsigned-integers) | [Primitive](#basic-data-types) | -| `30-37` | [`float`](#floats) | [Primitive](#basic-data-types) | -| `3A-3C` | [`decimal`](#floats) | [Primitive](#basic-data-types) | -| `3F` | [`bfloat16`](#floats) | [Primitive](#basic-data-types) | -| `70-7F` | [`array`](#array) | [Composite](#composite-data-types) | -| `80-8F` | [`list`](#list) | [Composite](#composite-data-types) | -| `D0-D2` | [`dictionary`](#dictionary) | [Composite](#composite-data-types) | -| `D3-D5` | [`map`](#map) | [Composite](#composite-data-types) | -| `DA-DC` | [`array`](#array) | [Composite](#composite-data-types) | -| `DD-DF` | [`list`](#list) | [Composite](#composite-data-types) | -| `E0` | [`string`](#string) | [Meta](#meta-data-types) | -| `E3` | [`vector`](#vector) | [Meta](#meta-data-types) | -| `F0` | [`optional`](#optional) | [Meta](#meta-data-types) | -| `F1` | [`enum`](#enum) | [Meta](#meta-data-types) | -| `FF` | [`error`](#error) | [Meta](#meta-data-types) | \ No newline at end of file diff --git a/representation.png b/representation.png deleted file mode 100644 index 0360800..0000000 Binary files a/representation.png and /dev/null differ diff --git a/zig/build.zig b/zig/build.zig index b8864d5..233990c 100644 --- a/zig/build.zig +++ b/zig/build.zig @@ -4,20 +4,33 @@ pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); - const config_options = b.addOptions(); - config_options.addOption(u8, "HBP_VERSION", 1); + const exe_mod = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }); const exe = b.addExecutable(.{ .name = "hbp", - .root_source_file = b.path("src/main.zig"), + .root_module = exe_mod, + }); + + b.installArtifact(exe); + + const hbp = b.addModule("hbp", .{ + .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, }); - b.installArtifact(exe); + const libhbp = b.addLibrary(.{ + .name = "hbp", + .root_module = hbp, + }); + + b.installArtifact(libhbp); const run_cmd = b.addRunArtifact(exe); - exe.root_module.addOptions("config", config_options); run_cmd.step.dependOn(b.getInstallStep()); const run_step = b.step("run", "Run the app"); @@ -26,13 +39,7 @@ pub fn build(b: *std.Build) void { const check = b.step("check", "Check if it compiles"); check.dependOn(&exe.step); - const exe_test = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - exe_test.root_module.addOptions("config", config_options); + const exe_test = b.addTest(.{ .root_module = exe_mod }); const test_artifact = b.addRunArtifact(exe_test); const test_step = b.step("test", "Run unit tests on the exports"); diff --git a/zig/build.zig.zon b/zig/build.zig.zon index 384c630..4de03bf 100644 --- a/zig/build.zig.zon +++ b/zig/build.zig.zon @@ -1,7 +1,8 @@ .{ .fingerprint = 0x8af3a6b7cf3baeb7, .name = .libhbp, - .version = "0.0.0", + .version = "0.1.0", + .minimum_zig_version = "0.16.0", .paths = .{ "build.zig", "build.zig.zon", diff --git a/zig/src/Deserializer.zig b/zig/src/Deserializer.zig deleted file mode 100644 index fe2b5a9..0000000 --- a/zig/src/Deserializer.zig +++ /dev/null @@ -1,85 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const HBP_VERSION = @import("config").HBP_VERSION; - -const assert = std.debug.assert; -const expect = std.testing.expect; - -const native_endian = builtin.cpu.arch.endian(); - -const Deserializer = @This(); - -pub fn deserializeBool(buffer: []const u8) bool { - return switch (buffer[1]) { - 0x01 => false, - 0x02 => true, - else => unreachable, - }; -} - -test deserializeBool { - try expect(deserializeBool(&.{ 0x01, 0x01 }) == false); - try expect(deserializeBool(&.{ 0x01, 0x02 }) == true); -} - -/// Fast path deserialization for integers. -/// Skips checks on the type and assumes it will be the same -/// or fit in the given integer type. -pub fn deserializeIntAssumeType(comptime T: type, buffer: []const u8) std.math.ByteAlignedInt(T) { - const alignedType = std.math.ByteAlignedInt(T); - const info = @typeInfo(alignedType).int; - - const base = if (comptime info.signedness == .signed) 0x10 else 0x20; - const buf = if (buffer[0] == HBP_VERSION) buffer[1..] else buffer; - - switch (buf[0]) { - base...(base + 6) => { - const slice = buf[1..]; - assert(slice.len <= (info.bits / 8)); - const data = std.mem.bytesToValue(alignedType, slice); - return std.mem.nativeToBig(alignedType, data); - }, - base + 0x0F => { - const slice = buf[3..]; - assert(slice.len <= (info.bits / 8)); - const data = std.mem.bytesToValue(alignedType, slice); - return std.mem.nativeToBig(alignedType, data); - }, - else => unreachable, - } -} - -pub fn deserializeStringAssumeLength(comptime max_len: u32, buffer: []const u8) !struct { [max_len]u8, u32 } { - var buf = if (buffer[0] == HBP_VERSION) buffer[1..] else buffer; - if (buf[0] != 0xE0) return error.InvalidBuffer; - - buf = buf[3..]; - - var temp: [max_len]u8 = undefined; - @memcpy(temp[0..buf.len], buf); - return .{ temp, @intCast(buf.len) }; -} - -test deserializeIntAssumeType { - try expect(deserializeIntAssumeType(i8, &.{ 0x01, 0x10, 0x2D }) == 45); - try expect(deserializeIntAssumeType(i16, &.{ 0x01, 0x11, 0x18, 0xCB }) == 6347); - try expect(deserializeIntAssumeType(i32, &.{ 0x01, 0x12, 0x00, 0x8B, 0x36, 0x60 }) == 9123424); - try expect(deserializeIntAssumeType(i64, &.{ 0x01, 0x13, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF }) == 5294967295); - try expect(deserializeIntAssumeType(i128, &.{ 0x01, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8A, 0xC7, 0x23, 0x04, 0x89, 0xE7, 0xFF, 0xFF }) == 28446744073709551615); - try expect(deserializeIntAssumeType(i256, &.{ 0x01, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x8D, 0x7E, 0xA4, 0xC6, 0x80, 0x00 }) == 340282366920938463463375607431768211456); - try expect(deserializeIntAssumeType(i512, &.{ 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x75, 0xD6, 0x95, 0xC2, 0x70, 0x6A, 0xC5, 0xE9, 0x70, 0x44, 0xC3, 0xB2, 0xD3, 0xEF, 0x59, 0x29, 0x94, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }) == 115792089237316395423570985008687907853269984665640564039457584007913129639935); - try expect(deserializeIntAssumeType(i6, &.{ 0x01, 0x10, 0x1E }) == 30); - try expect(deserializeIntAssumeType(i38, &.{ 0x01, 0x1F, 0x00, 0x28, 0x00, 0x00, 0x8B, 0x36, 0x60 }) == 9123424); - try expect(deserializeIntAssumeType(i80, &.{ 0x01, 0x1F, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF }) == 5294967295); - - try expect(deserializeIntAssumeType(u8, &.{ 0x01, 0x20, 0xFA }) == 250); - try expect(deserializeIntAssumeType(u16, &.{ 0x01, 0x21, 0x18, 0xCB }) == 6347); - try expect(deserializeIntAssumeType(u32, &.{ 0x01, 0x22, 0x00, 0x8B, 0x36, 0x60 }) == 9123424); - try expect(deserializeIntAssumeType(u64, &.{ 0x01, 0x23, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF }) == 5294967295); - try expect(deserializeIntAssumeType(u128, &.{ 0x01, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8A, 0xC7, 0x23, 0x04, 0x89, 0xE7, 0xFF, 0xFF }) == 28446744073709551615); - try expect(deserializeIntAssumeType(u256, &.{ 0x01, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x8D, 0x7E, 0xA4, 0xC6, 0x80, 0x00 }) == 340282366920938463463375607431768211456); - try expect(deserializeIntAssumeType(u512, &.{ 0x01, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x75, 0xD6, 0x95, 0xC2, 0x70, 0x6A, 0xC5, 0xE9, 0x70, 0x44, 0xC3, 0xB2, 0xD3, 0xEF, 0x59, 0x29, 0x94, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }) == 115792089237316395423570985008687907853269984665640564039457584007913129639935); - try expect(deserializeIntAssumeType(u6, &.{ 0x01, 0x20, 0x1E }) == 30); - try expect(deserializeIntAssumeType(u38, &.{ 0x01, 0x2F, 0x00, 0x28, 0x00, 0x00, 0x8B, 0x36, 0x60 }) == 9123424); - try expect(deserializeIntAssumeType(u80, &.{ 0x01, 0x2F, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF }) == 5294967295); -} diff --git a/zig/src/Scanner.zig b/zig/src/Scanner.zig new file mode 100644 index 0000000..c69e03f --- /dev/null +++ b/zig/src/Scanner.zig @@ -0,0 +1,611 @@ +const std = @import("std"); + +pub const Scanner = @This(); + +state: State = .identifier, +value_start: usize = undefined, +input: []const u8 = undefined, +cursor: usize = 0, + +/// Allocator used purely for the bitstack +pub fn init(input: []const u8) Scanner { + return .{ + .input = input, + }; +} + +pub fn deinit(self: *Scanner) void { + self.* = undefined; +} + +pub fn peekNextTokenType(self: *Scanner) !TokenType { + switch (self.state) { + .marker, .post_value => { + const m = std.enums.fromInt(MarkerType, self.input[self.cursor]) orelse return error.UnknownMarker; + return switch (m) { + .null => .null, + .false => .false, + .true => .true, + .signed_int_8, + .signed_int_16, + .signed_int_32, + .signed_int64, + .signed_int_128, + .signed_int_256, + .signed_int_512, + .arbitrary_signed_int, + .unsigned_int_8, + .unsigned_int_16, + .unsigned_int_32, + .unsigned_int64, + .unsigned_int_128, + .unsigned_int_256, + .unsigned_int_512, + .arbitrary_unsigned_int, + => .int, + .half_float, + // .minifloat, + .single_float, + // .extended_float_40, + .double_float, + .extended_float_80, + .quadruple_float, + // .octuple_float, + // .brain_float, + => .float, + .empty_string, + .string_1, + .string_2, + .string_3, + .string_4, + .string_5, + .string_6, + .string_7, + .string_8, + .string_9, + .string_10, + .string_11, + .string_12, + .string_13, + .string_14, + .string_15, + .arbitrary_string_1, + .arbitrary_string_2, + .arbitrary_string_4, + => .string, + .empty_tuple, + .tuple_1, + .tuple_2, + .tuple_3, + .tuple_4, + .tuple_5, + .tuple_6, + .tuple_7, + .tuple_8, + .tuple_9, + .tuple_10, + .tuple_11, + .tuple_12, + .tuple_13, + .tuple_14, + .tuple_15, + .arbitrary_tuple_1, + .arbitrary_tuple_2, + .arbitrary_tuple_4, + => .tuple, + .empty_vector, + .vector_1, + .vector_2, + .vector_3, + .vector_4, + .vector_5, + .vector_6, + .vector_7, + .vector_8, + .vector_9, + .vector_10, + .vector_11, + .vector_12, + .vector_13, + .vector_14, + .vector_15, + .arbitrary_vector_1, + .arbitrary_vector_2, + .arbitrary_vector_4, + => .vector, + .arbitrary_dict_1, + .arbitrary_dict_2, + .arbitrary_dict_4, + => .@"struct", + .optional => .optional, + .@"enum" => .@"enum", + .@"union" => .@"union", + else => error.NotImplemented, + }; + }, + else => return error.UnsupportedLookup, + } +} + +pub fn next(self: *Scanner) !Token { + state: switch (self.state) { + .identifier => { + self.cursor += 1; + self.state = .marker; + return .{ .identifier = self.input[0] }; + }, + .marker => { + const m = std.enums.fromInt(MarkerType, self.input[self.cursor]) orelse return error.UnknownMarker; + switch (m) { + .optional => { + self.cursor += 1; + return .optional; + }, + .null => { + self.cursor += 1; + self.state = .post_value; + return .null; + }, + .false => { + self.cursor += 1; + self.state = .post_value; + return .false; + }, + .true => { + self.cursor += 1; + self.state = .post_value; + return .true; + }, + .signed_int_8, + .signed_int_16, + .signed_int_32, + .signed_int64, + .signed_int_128, + .signed_int_256, + .signed_int_512, + => { + self.state = .int; + continue :state .int; + }, + .arbitrary_signed_int => { + self.cursor += 1; + self.state = .arbitrary_int; + continue :state .arbitrary_int; + }, + .unsigned_int_8, + .unsigned_int_16, + .unsigned_int_32, + .unsigned_int64, + .unsigned_int_128, + .unsigned_int_256, + .unsigned_int_512, + => { + self.state = .uint; + continue :state .uint; + }, + .arbitrary_unsigned_int => { + self.cursor += 1; + self.state = .arbitrary_uint; + continue :state .arbitrary_uint; + }, + .half_float => { + self.cursor += 3; + self.state = .post_value; + return .{ .float = .{ .bits = 16, .view = self.input[self.cursor - 2 .. self.cursor] } }; + }, + .single_float => { + self.cursor += 5; + self.state = .post_value; + return .{ .float = .{ .bits = 32, .view = self.input[self.cursor - 4 .. self.cursor] } }; + }, + .double_float => { + self.cursor += 9; + self.state = .post_value; + return .{ .float = .{ .bits = 64, .view = self.input[self.cursor - 8 .. self.cursor] } }; + }, + .extended_float_80 => { + self.cursor += 11; + self.state = .post_value; + return .{ .float = .{ .bits = 80, .view = self.input[self.cursor - 10 .. self.cursor] } }; + }, + .quadruple_float => { + self.cursor += 17; + self.state = .post_value; + return .{ .float = .{ .bits = 128, .view = self.input[self.cursor - 16 .. self.cursor] } }; + }, + .minifloat, + .extended_float_40, + .octuple_float, + .brain_float, + => { + return error.FloatFormatNotImplemented; + }, + .decimal_32, + .decimal_64, + .decimal_128, + => { + return error.DecimalsNotImplemented; + }, + .empty_string, + .string_1, + .string_2, + .string_3, + .string_4, + .string_5, + .string_6, + .string_7, + .string_8, + .string_9, + .string_10, + .string_11, + .string_12, + .string_13, + .string_14, + .string_15, + => { + const byte_length = self.input[self.cursor] - 0x60; + self.cursor += 1; + return self.parseStringState(byte_length); + }, + // TODO: The spec for arbitrary strings is a bit more complex than this + // But since its not yet documented we aren't implementing it fully + .arbitrary_string_1 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u8, self.input[self.cursor]); + self.cursor += 1; + return self.parseStringState(byte_length); + }, + .arbitrary_string_2 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u16, std.mem.bytesToValue(u16, self.input[self.cursor .. self.cursor + 2])); + self.cursor += 2; + return self.parseStringState(byte_length); + }, + .arbitrary_string_4 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u32, std.mem.bytesToValue(u32, self.input[self.cursor .. self.cursor + 4])); + self.cursor += 4; + return self.parseStringState(byte_length); + }, + .empty_tuple, + .tuple_1, + .tuple_2, + .tuple_3, + .tuple_4, + .tuple_5, + .tuple_6, + .tuple_7, + .tuple_8, + .tuple_9, + .tuple_10, + .tuple_11, + .tuple_12, + .tuple_13, + .tuple_14, + .tuple_15, + => { + const byte_length = self.input[self.cursor] - 0x70; + self.cursor += 1; + return self.parseTupleState(byte_length); + }, + .arbitrary_tuple_1 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u8, self.input[self.cursor]); + self.cursor += 1; + return self.parseTupleState(byte_length); + }, + .arbitrary_tuple_2 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u16, std.mem.bytesToValue(u16, self.input[self.cursor .. self.cursor + 2])); + self.cursor += 2; + return self.parseTupleState(byte_length); + }, + .arbitrary_tuple_4 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u32, std.mem.bytesToValue(u32, self.input[self.cursor .. self.cursor + 4])); + self.cursor += 4; + return self.parseTupleState(byte_length); + }, + .empty_vector, + .vector_1, + .vector_2, + .vector_3, + .vector_4, + .vector_5, + .vector_6, + .vector_7, + .vector_8, + .vector_9, + .vector_10, + .vector_11, + .vector_12, + .vector_13, + .vector_14, + .vector_15, + => { + const byte_length = self.input[self.cursor] - 0x80; + self.cursor += 1; + return self.parseVectorState(byte_length); + }, + .arbitrary_vector_1 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u8, self.input[self.cursor]); + self.cursor += 1; + return self.parseVectorState(byte_length); + }, + .arbitrary_vector_2 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u16, std.mem.bytesToValue(u16, self.input[self.cursor .. self.cursor + 2])); + self.cursor += 2; + return self.parseVectorState(byte_length); + }, + .arbitrary_vector_4 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u32, std.mem.bytesToValue(u32, self.input[self.cursor .. self.cursor + 4])); + self.cursor += 4; + return self.parseVectorState(byte_length); + }, + .arbitrary_dict_1 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u8, self.input[self.cursor]); + self.cursor += 1; + return self.parseStructState(byte_length); + }, + .arbitrary_dict_2 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u16, std.mem.bytesToValue(u16, self.input[self.cursor .. self.cursor + 2])); + self.cursor += 2; + return self.parseStructState(byte_length); + }, + .arbitrary_dict_4 => { + self.cursor += 1; + const byte_length = std.mem.nativeToLittle(u32, std.mem.bytesToValue(u32, self.input[self.cursor .. self.cursor + 4])); + self.cursor += 4; + return self.parseStructState(byte_length); + }, + //? TODO?: Could be worth optimizing u1 enums in the sense of booleans, this has to be discussed further + // NOTE: Maybe it could be worth to allow enums to omit the type marker for `u8` enums + .@"enum" => { + self.cursor += 1; + if (self.input[self.cursor] == @intFromEnum(MarkerType.arbitrary_unsigned_int)) { + self.cursor += 1; + self.state = .arbitrary_uint; + } else { + self.state = .uint; + } + + return .@"enum"; + }, + .@"union" => { + self.cursor += 1; + return .@"union"; + }, + else => return error.NotImplemented, + } + }, + .int, .uint => { + const marker = self.input[self.cursor]; + self.cursor += 1; + const byte_length = try calculateIntegerByteLength(marker); + const value_start = self.cursor; + self.cursor += byte_length; + const state = self.state; + self.state = .post_value; + + return .{ + .int = .{ + .signedness = if (state == .int) .signed else .unsigned, + .view = self.input[value_start..self.cursor], + }, + }; + }, + .arbitrary_int, .arbitrary_uint => { + const byte_length = std.mem.nativeToLittle(u16, std.mem.bytesToValue(u16, self.input[self.cursor .. self.cursor + 2])); + self.cursor += 2; + const value_start = self.cursor; + self.cursor += byte_length; + const state = self.state; + self.state = .post_value; + + return .{ + .int = .{ + .signedness = if (state == .arbitrary_int) .signed else .unsigned, + .view = self.input[value_start..self.cursor], + }, + }; + }, + .post_value => { + if (self.checkEnd()) return .eos; + // NOTE: If no changes are necessary here after all types have been implemented + // remove this and add a simple check to the beginning of `marker` instead. + self.state = .marker; + continue :state .marker; + }, + } +} + +fn calculateIntegerByteLength(marker: u8) !usize { + return switch (marker) { + 0x10...0x16 => std.math.pow(usize, 2, (marker - 0x10)), + 0x20...0x26 => std.math.pow(usize, 2, (marker - 0x20)), + else => error.InvalidMarker, + }; +} + +fn checkEnd(self: *Scanner) bool { + return self.cursor >= self.input.len; +} + +fn parseStringState(self: *Scanner, byte_length: u32) Token { + self.state = if (byte_length == 0) .post_value else .marker; + return .{ .string = byte_length }; +} + +fn parseTupleState(self: *Scanner, byte_length: u32) Token { + self.state = if (byte_length == 0) .post_value else .marker; + return .{ .tuple = byte_length }; +} + +fn parseVectorState(self: *Scanner, elements: u32) Token { + self.state = if (elements == 0) .post_value else .marker; + return .{ .vector = elements }; +} + +fn parseStructState(self: *Scanner, kv_pairs: u32) Token { + self.state = if (kv_pairs == 0) .post_value else .marker; + return .{ .@"struct" = kv_pairs }; +} + +pub const MarkerType = enum(u8) { + null = 0x00, + false = 0x01, + true = 0x02, + signed_int_8 = 0x10, + signed_int_16 = 0x11, + signed_int_32 = 0x12, + signed_int64 = 0x13, + signed_int_128 = 0x14, + signed_int_256 = 0x15, + signed_int_512 = 0x16, + arbitrary_signed_int = 0x1F, + unsigned_int_8 = 0x20, + unsigned_int_16 = 0x21, + unsigned_int_32 = 0x22, + unsigned_int64 = 0x23, + unsigned_int_128 = 0x24, + unsigned_int_256 = 0x25, + unsigned_int_512 = 0x26, + arbitrary_unsigned_int = 0x2F, + half_float = 0x30, + minifloat = 0x31, + single_float = 0x32, + extended_float_40 = 0x33, + double_float = 0x34, + extended_float_80 = 0x35, + quadruple_float = 0x36, + octuple_float = 0x37, + brain_float = 0x3F, + decimal_32 = 0x3A, + decimal_64 = 0x3B, + decimal_128 = 0x3C, + empty_string = 0x60, + string_1 = 0x61, + string_2 = 0x62, + string_3 = 0x63, + string_4 = 0x64, + string_5 = 0x65, + string_6 = 0x66, + string_7 = 0x67, + string_8 = 0x68, + string_9 = 0x69, + string_10 = 0x6A, + string_11 = 0x6B, + string_12 = 0x6C, + string_13 = 0x6D, + string_14 = 0x6E, + string_15 = 0x6F, + arbitrary_string_1 = 0xC0, + arbitrary_string_2 = 0xC1, + arbitrary_string_4 = 0xC2, + empty_tuple = 0x70, + tuple_1 = 0x71, + tuple_2 = 0x72, + tuple_3 = 0x73, + tuple_4 = 0x74, + tuple_5 = 0x75, + tuple_6 = 0x76, + tuple_7 = 0x77, + tuple_8 = 0x78, + tuple_9 = 0x79, + tuple_10 = 0x7A, + tuple_11 = 0x7B, + tuple_12 = 0x7C, + tuple_13 = 0x7D, + tuple_14 = 0x7E, + tuple_15 = 0x7F, + arbitrary_tuple_1 = 0xDA, + arbitrary_tuple_2 = 0xDB, + arbitrary_tuple_4 = 0xDC, + empty_vector = 0x80, + vector_1 = 0x81, + vector_2 = 0x82, + vector_3 = 0x83, + vector_4 = 0x84, + vector_5 = 0x85, + vector_6 = 0x86, + vector_7 = 0x87, + vector_8 = 0x88, + vector_9 = 0x89, + vector_10 = 0x8A, + vector_11 = 0x8B, + vector_12 = 0x8C, + vector_13 = 0x8D, + vector_14 = 0x8E, + vector_15 = 0x8F, + arbitrary_vector_1 = 0xDD, + arbitrary_vector_2 = 0xDE, + arbitrary_vector_4 = 0xDF, + + // TODO: Have optimized markers like lists + arbitrary_dict_1 = 0xD0, + arbitrary_dict_2 = 0xD1, + arbitrary_dict_4 = 0xD2, + optional = 0xE0, + @"enum" = 0xE1, + @"union" = 0xE2, + @"error" = 0xFF, +}; + +pub const State = enum { + identifier, + marker, + post_value, + int, + arbitrary_int, + uint, + arbitrary_uint, +}; + +pub const TokenType = enum { + identifier, + null, + false, + true, + int, + float, + string, + tuple, + vector, + @"struct", + optional, + @"enum", + @"union", + eos, +}; + +pub const Token = union(TokenType) { + identifier: u8, + null, + false, + true, + int: struct { + signedness: std.builtin.Signedness, + view: []const u8, + }, + float: struct { + bits: u16, + view: []const u8, + }, + string: u32, + tuple: u32, + vector: u32, + @"struct": u32, + optional, + @"enum", + @"union", + eos, +}; + +test { + _ = @import("./scanner_test.zig"); +} diff --git a/zig/src/Serializer.zig b/zig/src/Serializer.zig deleted file mode 100644 index e986eec..0000000 --- a/zig/src/Serializer.zig +++ /dev/null @@ -1,186 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const HBP_VERSION = @import("config").HBP_VERSION; - -const ByteAlignedInt = std.math.ByteAlignedInt; - -const assert = std.debug.assert; -const expect = std.testing.expect; -const eql = std.mem.eql; - -const native_endian = builtin.cpu.arch.endian(); - -const Serializer = @This(); - -pub fn calculateMarker(comptime T: type) []const u8 { - comptime { - const type_info = @typeInfo(T); - - switch (type_info) { - .int => { - const alignedType = ByteAlignedInt(T); - const info = @typeInfo(alignedType).int; - const base = if (info.signedness == .signed) 0x10 else 0x20; - return switch (info.bits) { - 8 => &.{base}, - 16 => &.{base + 1}, - 32 => &.{base + 2}, - 64 => &.{base + 3}, - 128 => &.{base + 4}, - 256 => &.{base + 5}, - 512 => &.{base + 6}, - else => |bits| &[_]u8{base + 0x0F} ++ &@as([2]u8, @bitCast(if (native_endian == .big) bits else @byteSwap(bits))), - }; - }, - .array => |arr| { - const is_string = arr.sentinel() != null; - const base = 0x80; - const marker = blk: { - if (arr.len <= 15) break :blk [_]u8{base + arr.len} ++ calculateMarker(arr.child); - @compileError("Not yet implemented"); - }; - if (is_string) return [_]u8{0xE0} ++ marker; - return marker; - }, - else => unreachable, - } - } -} - -pub fn calculateBufferLen(comptime T: type) comptime_int { - comptime { - const type_info = @typeInfo(T); - - switch (type_info) { - .int => |i| { - return @divExact(i.bits, 8); - }, - .array => |arr| { - return calculateBufferLen(arr.child) * arr.len; - }, - else => unreachable, - } - } -} - -pub fn Buffer(comptime T: type) type { - comptime { - const type_info = @typeInfo(T); - - return switch (type_info) { - .null, .bool => [2]u8, - .int => [1 + calculateMarker(T).len + calculateBufferLen(ByteAlignedInt(T))]u8, - .array => [1 + calculateMarker(T).len + calculateBufferLen(T)]u8, - else => unreachable, - }; - } -} - -pub fn serializeNull() Buffer(@TypeOf(null)) { - var buffer: Buffer(@TypeOf(null)) = undefined; - buffer[0] = HBP_VERSION; - buffer[1] = 0x00; - return buffer; -} - -test serializeNull { - try expect(eql(u8, &serializeNull(), &.{ 0x01, 0x00 })); -} - -pub fn serializeBool(value: bool) Buffer(bool) { - var buffer: Buffer(@TypeOf(null)) = undefined; - buffer[0] = HBP_VERSION; - buffer[1] = if (value) 0x02 else 0x01; - return buffer; -} - -test serializeBool { - try expect(eql(u8, &serializeBool(false), &.{ 0x01, 0x01 })); - try expect(eql(u8, &serializeBool(true), &.{ 0x01, 0x02 })); -} - -pub fn serializeInt(comptime T: type, value: T) Buffer(T) { - const aligned_type = ByteAlignedInt(T); - const marker = comptime calculateMarker(T); - - var buffer: Buffer(T) = undefined; - buffer[0] = HBP_VERSION; - - inline for (marker, 1..) |byte, i| { - buffer[i] = byte; - } - - buffer[1 + marker.len ..].* = @bitCast(std.mem.nativeToBig(aligned_type, value)); - - return buffer; -} - -test serializeInt { - try expect(eql(u8, &serializeInt(i8, 45), &.{ 0x01, 0x10, 0x2D })); - try expect(eql(u8, &serializeInt(i16, 6347), &.{ 0x01, 0x11, 0x18, 0xCB })); - try expect(eql(u8, &serializeInt(i32, 9123424), &.{ 0x01, 0x12, 0x00, 0x8B, 0x36, 0x60 })); - try expect(eql(u8, &serializeInt(i64, 5294967295), &.{ 0x01, 0x13, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF })); - try expect(eql(u8, &serializeInt(i128, 28446744073709551615), &.{ 0x01, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8A, 0xC7, 0x23, 0x04, 0x89, 0xE7, 0xFF, 0xFF })); - try expect(eql(u8, &serializeInt(i256, 340282366920938463463375607431768211456), &.{ 0x01, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x8D, 0x7E, 0xA4, 0xC6, 0x80, 0x00 })); - try expect(eql(u8, &serializeInt(i512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), &.{ 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x75, 0xD6, 0x95, 0xC2, 0x70, 0x6A, 0xC5, 0xE9, 0x70, 0x44, 0xC3, 0xB2, 0xD3, 0xEF, 0x59, 0x29, 0x94, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF })); - try expect(eql(u8, &serializeInt(i6, 30), &.{ 0x01, 0x10, 0x1E })); - try expect(eql(u8, &serializeInt(i38, 9123424), &.{ 0x01, 0x1F, 0x00, 0x28, 0x00, 0x00, 0x8B, 0x36, 0x60 })); - try expect(eql(u8, &serializeInt(i80, 5294967295), &.{ 0x01, 0x1F, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF })); - - try expect(eql(u8, &serializeInt(u8, 250), &.{ 0x01, 0x20, 0xFA })); - try expect(eql(u8, &serializeInt(u16, 6347), &.{ 0x01, 0x21, 0x18, 0xCB })); - try expect(eql(u8, &serializeInt(u32, 9123424), &.{ 0x01, 0x22, 0x00, 0x8B, 0x36, 0x60 })); - try expect(eql(u8, &serializeInt(u64, 5294967295), &.{ 0x01, 0x23, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF })); - try expect(eql(u8, &serializeInt(u128, 28446744073709551615), &.{ 0x01, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8A, 0xC7, 0x23, 0x04, 0x89, 0xE7, 0xFF, 0xFF })); - try expect(eql(u8, &serializeInt(u256, 340282366920938463463375607431768211456), &.{ 0x01, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x8D, 0x7E, 0xA4, 0xC6, 0x80, 0x00 })); - try expect(eql(u8, &serializeInt(u512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), &.{ 0x01, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x75, 0xD6, 0x95, 0xC2, 0x70, 0x6A, 0xC5, 0xE9, 0x70, 0x44, 0xC3, 0xB2, 0xD3, 0xEF, 0x59, 0x29, 0x94, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF })); - try expect(eql(u8, &serializeInt(u6, 30), &.{ 0x01, 0x20, 0x1E })); - try expect(eql(u8, &serializeInt(u38, 9123424), &.{ 0x01, 0x2F, 0x00, 0x28, 0x00, 0x00, 0x8B, 0x36, 0x60 })); - try expect(eql(u8, &serializeInt(u80, 5294967295), &.{ 0x01, 0x2F, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3B, 0x9A, 0xC9, 0xFF })); -} - -pub fn serializeList(comptime T: type, comptime max_len: u32, value: []const T) struct { Buffer([max_len]T), u32 } { - assert(value.len <= max_len); - const type_info = @typeInfo(T); - - // TODO: Support more types - if (type_info != .int) @compileError("Only integer lists are supported"); - - var buffer: Buffer([max_len]T) = undefined; - buffer[0] = HBP_VERSION; - - const marker = comptime calculateMarker([max_len]T); - - inline for (marker, 1..) |byte, i| { - buffer[i] = byte; - } - - const byte_size = calculateBufferLen(T); - var real_length: u32 = 1 + marker.len; - - for (value) |el| { - @memcpy(buffer[real_length .. real_length + byte_size], &@as([byte_size]u8, @bitCast(std.mem.nativeToBig(T, el)))); - real_length += byte_size; - } - - return .{ buffer, real_length }; -} - -// TODO: Merge this properly with `serializeList` -pub fn serializeString(comptime max_len: u32, value: []const u8) struct { Buffer([max_len:0]u8), u32 } { - assert(value.len <= max_len); - var buffer: Buffer([max_len:0]u8) = undefined; - buffer[0] = HBP_VERSION; - - const marker = comptime calculateMarker([max_len:0]u8); - - inline for (marker, 1..) |byte, i| { - buffer[i] = byte; - } - - const initial_len: usize = 1 + marker.len; - - @memcpy(buffer[initial_len .. initial_len + value.len], value); - - return .{ buffer, @intCast(initial_len + value.len) }; -} diff --git a/zig/src/main.zig b/zig/src/main.zig index f07a19b..f9f758b 100644 --- a/zig/src/main.zig +++ b/zig/src/main.zig @@ -1,14 +1,85 @@ const std = @import("std"); -const Serializer = @import("./Serializer.zig"); -const Deserializer = @import("./Deserializer.zig"); +const Serializer = @import("./serializer.zig"); +const Deserializer = @import("./static.zig"); +const Scanner = @import("./Scanner.zig"); + +const test_bool = [_][]const u8{ + Serializer.serializeComptime(bool, false), + Serializer.serializeComptime(bool, true), +}; + +const test_int = [_][]const u8{ + Serializer.serializeComptime(i8, 45), + Serializer.serializeComptime(i16, 6347), + Serializer.serializeComptime(i32, 9123424), + Serializer.serializeComptime(i64, 5294967295), + Serializer.serializeComptime(i128, 28446744073709551615), + Serializer.serializeComptime(i256, 340282366920938463463375607431768211456), + Serializer.serializeComptime(i512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), + Serializer.serializeComptime(i6, 30), + Serializer.serializeComptime(i38, 9123424), + Serializer.serializeComptime(i80, 5294967295), +}; +const test_uint = [_][]const u8{ + Serializer.serializeComptime(u8, 250), + Serializer.serializeComptime(u16, 6347), + Serializer.serializeComptime(u32, 9123424), + Serializer.serializeComptime(u64, 5294967295), + Serializer.serializeComptime(u128, 28446744073709551615), + Serializer.serializeComptime(u256, 340282366920938463463375607431768211456), + Serializer.serializeComptime(u512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), + Serializer.serializeComptime(u6, 30), + Serializer.serializeComptime(u38, 9123424), + Serializer.serializeComptime(u80, 5294967295), +}; + +const test_float = [_][]const u8{ + Serializer.serializeComptime(f16, 20.11), + Serializer.serializeComptime(f32, 202.5456), + Serializer.serializeComptime(f64, 220.563564), + Serializer.serializeComptime(f80, 25460.565667657), + Serializer.serializeComptime(f128, 2024.5689076899345), +}; pub fn main() !void { - const serialized, const length = Serializer.serializeString(15, "Hello World"); - const deserialized, const len = try Deserializer.deserializeStringAssumeLength(15, serialized[0..length]); + var allocator: std.heap.DebugAllocator(.{}) = .init; + const gpa = allocator.allocator(); + + for (test_bool) |in| { + print(bool, gpa, in); + } + for (test_int) |in| { + print(i512, gpa, in); + } + for (test_uint) |in| { + print(u512, gpa, in); + } + for (test_float) |in| { + print(f128, gpa, in); + } + + print(struct { u8, u16 }, gpa, comptime Serializer.serializeComptime(struct { u8, u16 }, .{ 36, 3204 })); + print(?u8, gpa, comptime Serializer.serializeComptime(?u8, 250)); + print(?u8, gpa, comptime Serializer.serializeComptime(?u8, null)); + print(enum { TEST }, gpa, comptime Serializer.serializeComptime(enum { TEST }, .TEST)); + print([]const u8, gpa, comptime Serializer.serializeComptime([]const u8, "hello world")); + print(struct { x: u32 }, gpa, comptime Serializer.serializeComptime(struct { x: u32 }, .{ .x = 43545 })); + print(union(enum(u1)) { x: u32, y: []const u8 }, gpa, comptime Serializer.serializeComptime(union(enum(u1)) { x: u32, y: []const u8 }, .{ .x = 760589 })); + print(union(enum(u1)) { x: u32, y: []const u8 }, gpa, comptime Serializer.serializeComptime(union(enum(u1)) { x: u32, y: []const u8 }, .{ .y = "yo world" })); + print([]u8, gpa, comptime Serializer.serializeComptime([]u8, @constCast(@as([]const u8, &.{ 10, 60, 134 })))); + const t = struct { x: u32 }; + print([]t, gpa, comptime Serializer.serializeComptime([]t, @constCast(@as([]const t, &.{ .{ .x = 10 }, .{ .x = 60 }, .{ .x = 134 } })))); + print(packed struct(u64) { t: bool, z: bool, _: u62 }, gpa, comptime Serializer.serializeComptime(packed struct(u64) { t: bool, z: bool, _: u62 = 0 }, .{ .t = false, .z = true })); + print([3]t, gpa, comptime Serializer.serializeComptime([3]t, .{ .{ .x = 10 }, .{ .x = 60 }, .{ .x = 134 } })); + print([10]?t, gpa, comptime Serializer.serializeComptime([3]?t, .{ .{ .x = 10 }, .{ .x = 60 }, .{ .x = 134 } })); +} - var buf2: [64]u8 = undefined; - std.debug.print("HBP payload: {s} ({s})\n", .{ readableOutput(&buf2, serialized[0..length]), deserialized[0..len] }); - // std.debug.print("HBP payload: {s}\n", .{readableOutput(&buf2, serialized[0..length])}); +fn print(comptime T: type, gpa: std.mem.Allocator, payload: []const u8) void { + var buf: [512]u8 = undefined; + std.debug.print("-------------------------------------------------\n", .{}); + std.debug.print("HBP payload: {s}\n", .{readableOutput(&buf, payload)}); + std.debug.print("Parsed Payload: {any}\n", .{Deserializer.parseFromSlice(T, payload, gpa, .{})}); + buf = undefined; } fn readableOutput(buffer: []u8, input: []const u8) []const u8 { diff --git a/zig/src/root.zig b/zig/src/root.zig new file mode 100644 index 0000000..eadaf9f --- /dev/null +++ b/zig/src/root.zig @@ -0,0 +1,10 @@ +const static = @import("./static.zig"); + +pub const ParseOptions = static.ParseOptions; +pub const parseFromSlice = static.parseFromSlice; +pub const parseFromTokenSource = static.parseFromTokenSource; +pub const innerParse = static.innerParse; + +pub const Scanner = @import("./Scanner.zig"); +pub const serialize = @import("./serializer.zig").serialize; +pub const serializeFromValue = @import("./serializer.zig").serializeFromValue; diff --git a/zig/src/scanner_test.zig b/zig/src/scanner_test.zig new file mode 100644 index 0000000..2494880 --- /dev/null +++ b/zig/src/scanner_test.zig @@ -0,0 +1,105 @@ +const std = @import("std"); +const Scanner = @import("Scanner.zig"); +const Serializer = @import("./serializer.zig"); +const Token = Scanner.Token; + +const expect = std.testing.expect; +const eql = std.mem.eql; + +test "null" { + var scanner: Scanner = .init(&.{ 0x01, 0x00 }); + defer scanner.deinit(); + + try expectNext(&scanner, .{ .identifier = 1 }); + try expectNext(&scanner, .null); + try expectNext(&scanner, .eos); +} + +test "bool" { + var scanner: Scanner = .init(&.{ 0x01, 0x01 }); + + try expectNext(&scanner, .{ .identifier = 1 }); + try expectNext(&scanner, .false); + try expectNext(&scanner, .eos); + + scanner.deinit(); + scanner = .init(&.{ 0x01, 0x02 }); + defer scanner.deinit(); + + try expectNext(&scanner, .{ .identifier = 1 }); + try expectNext(&scanner, .true); + try expectNext(&scanner, .eos); +} + +test "int" { + const test_input = [_][]const u8{ + Serializer.serializeComptime(i8, 45), + Serializer.serializeComptime(i16, 6347), + Serializer.serializeComptime(i32, 9123424), + Serializer.serializeComptime(i64, 5294967295), + Serializer.serializeComptime(i128, 28446744073709551615), + Serializer.serializeComptime(i256, 340282366920938463463375607431768211456), + Serializer.serializeComptime(i512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), + Serializer.serializeComptime(u8, 250), + Serializer.serializeComptime(u16, 6347), + Serializer.serializeComptime(u32, 9123424), + Serializer.serializeComptime(u64, 5294967295), + Serializer.serializeComptime(u128, 28446744073709551615), + Serializer.serializeComptime(u256, 340282366920938463463375607431768211456), + Serializer.serializeComptime(u512, 115792089237316395423570985008687907853269984665640564039457584007913129639935), + }; + const test_output = [_]Token{ + Token{ .int = .{ .signedness = .signed, .view = &.{0x2D} } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0xCB, 0x18 } } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0x60, 0x36, 0x8B, 0x00 } } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0xFF, 0xC9, 0x9A, 0x3B, 0x01, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0xFF, 0xFF, 0xE7, 0x89, 0x04, 0x23, 0xC7, 0x8A, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0x00, 0x80, 0xC6, 0xA4, 0x7E, 0x8D, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .signed, .view = &.{ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0x94, 0x29, 0x59, 0xEF, 0xD3, 0xB2, 0xC3, 0x44, 0x70, 0xE9, 0xC5, 0x6A, 0x70, 0xC2, 0x95, 0xD6, 0x75, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + + Token{ .int = .{ .signedness = .unsigned, .view = &.{0xFA} } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0xCB, 0x18 } } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0x60, 0x36, 0x8B, 0x00 } } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0xFF, 0xC9, 0x9A, 0x3B, 0x01, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0xFF, 0xFF, 0xE7, 0x89, 0x04, 0x23, 0xC7, 0x8A, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0x00, 0x80, 0xC6, 0xA4, 0x7E, 0x8D, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + Token{ .int = .{ .signedness = .unsigned, .view = &.{ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0x94, 0x29, 0x59, 0xEF, 0xD3, 0xB2, 0xC3, 0x44, 0x70, 0xE9, 0xC5, 0x6A, 0x70, 0xC2, 0x95, 0xD6, 0x75, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } } }, + }; + + for (test_input, 0..) |in, i| { + var scanner: Scanner = .init(in); + defer scanner.deinit(); + + const out = test_output[i]; + try expectNext(&scanner, .{ .identifier = 1 }); + try expectNext(&scanner, out); + try expectNext(&scanner, .eos); + } +} + +fn expectNext(scanner: *Scanner, expected_token: Scanner.Token) !void { + const token = try scanner.next(); + try std.testing.expectEqual(std.meta.activeTag(expected_token), std.meta.activeTag(token)); + + switch (expected_token) { + .int => |expected_value| { + try expect(eql(u8, expected_value.view, token.int.view)); + }, + .float => |expected_value| { + try expect(eql(u8, expected_value.view, token.int.view)); + }, + .identifier, + .null, + .false, + .true, + .tuple, + .optional, + .@"enum", + .string, + .@"struct", + .@"union", + .vector, + .eos, + => {}, + } +} diff --git a/zig/src/serializer.zig b/zig/src/serializer.zig new file mode 100644 index 0000000..3aa94b3 --- /dev/null +++ b/zig/src/serializer.zig @@ -0,0 +1,242 @@ +const std = @import("std"); +const MarkerType = @import("./Scanner.zig").MarkerType; + +const HBP_VERSION = 1; + +pub inline fn serializeComptime(comptime T: type, comptime value: T) []const u8 { + comptime { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + + serialize(T, value, &w) catch unreachable; + + const buffered = w.buffered(); + const x: [buffered.len]u8 = buffered[0..buffered.len].*; + return &x; + } +} + +pub fn serializeFromValue(gpa: std.mem.Allocator, value: anytype) ![]const u8 { + var w: std.Io.Writer.Allocating = .init(gpa); + try serialize(@TypeOf(value), value, &w.writer); + return try w.toOwnedSlice(); +} + +pub fn serialize(comptime T: type, value: T, writer: *std.Io.Writer) !void { + // TODO: Use proper identifier + try writer.writeByte(HBP_VERSION); + try innerSerialize(T, value, writer); +} + +fn innerSerialize(comptime T: type, value: T, writer: *std.Io.Writer) !void { + const type_info = @typeInfo(T); + + switch (type_info) { + .void => return, + .optional => |optional_info| { + try writer.writeByte(@intFromEnum(MarkerType.optional)); + if (value) |opt| { + try innerSerialize(optional_info.child, opt, writer); + } else { + try writer.writeByte(@intFromEnum(MarkerType.null)); + } + }, + .null => { + try writer.writeByte(@intFromEnum(MarkerType.null)); + }, + .bool => { + try writer.writeByte(0x01 + @as(u8, @intFromBool(value))); + }, + .int => { + const aligned_type = std.math.ByteAlignedInt(T); + const int = @typeInfo(aligned_type).int; + + const marker = getIntMarker(int); + try writer.writeByte(marker); + if (marker & 0x0F == 0x0F) try writer.writeAll(&@as([2]u8, @bitCast(std.mem.nativeToLittle(u16, @divExact(int.bits, 8))))); + try writeInt(aligned_type, value, writer); + }, + // TODO: How are we supporting our other float types? + .float => |float_info| { + try writer.writeByte(getFloatMarker(float_info)); + try writeFloat(T, value, writer); + }, + .pointer => |pointer_info| { + switch (pointer_info.size) { + .slice => { + if (pointer_info.is_const and pointer_info.child == u8) { + if (value.len <= 15) { + try writer.writeByte(@intFromEnum(MarkerType.empty_string) + @as(u8, @intCast(value.len))); + } else if (value.len <= std.math.maxInt(u8)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_string_1)); + try writeInt(u8, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u16)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_string_2)); + try writeInt(u16, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u32)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_string_4)); + try writeInt(u32, @intCast(value.len), writer); + } + + try writer.writeAll(value); + } else { + try parseArrayOrSlice(pointer_info.child, value, writer); + } + }, + else => @compileError("Unsupported pointer type"), + } + }, + .array => |array_info| { + try parseArrayOrSlice(array_info.child, &value, writer); + }, + .@"enum" => |enum_info| { + std.debug.assert(@typeInfo(enum_info.tag_type).int.signedness == .unsigned); + try writer.writeByte(@intFromEnum(MarkerType.@"enum")); + try innerSerialize(enum_info.tag_type, @intFromEnum(value), writer); + }, + .@"union" => |union_info| { + if (union_info.tag_type) |tag_type| { + inline for (union_info.fields) |field| { + if (value == @field(tag_type, field.name)) { + try writer.writeByte(@intFromEnum(MarkerType.@"union")); + try innerSerialize(@typeInfo(tag_type).@"enum".tag_type, @intFromEnum(value), writer); + try innerSerialize(field.type, @field(value, field.name), writer); + } + } + } else @compileError("Unable to parse non tagged union '" ++ @typeName(T) ++ "'"); + }, + .@"struct" => |struct_info| { + // Packed structs are encoded as their backing integer and treated as bitfields + // TODO: Maybe it could be a good idea to have a Meta type to indicate bitfields + if (struct_info.layout == .@"packed") { + try innerSerialize(struct_info.backing_integer.?, @bitCast(value), writer); + } else if (struct_info.is_tuple) { + if (struct_info.fields.len <= 15) { + try writer.writeByte(@intFromEnum(MarkerType.empty_tuple) + struct_info.fields.len); + } else if (struct_info.fields.len <= std.math.maxInt(u8)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_1)); + try writeInt(u8, @intCast(struct_info.fields.len), writer); + } else if (struct_info.fields.len <= std.math.maxInt(u16)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_2)); + try writeInt(u16, @intCast(struct_info.fields.len), writer); + } else if (struct_info.fields.len <= std.math.maxInt(u32)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_4)); + try writeInt(u32, @intCast(struct_info.fields.len), writer); + } + + inline for (struct_info.fields, 0..) |field, i| { + try innerSerialize(field.type, value[i], writer); + } + } else { + if (struct_info.fields.len <= std.math.maxInt(u8)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_dict_1)); + try writeInt(u8, @intCast(struct_info.fields.len), writer); + } else if (struct_info.fields.len <= std.math.maxInt(u16)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_dict_2)); + try writeInt(u16, @intCast(struct_info.fields.len), writer); + } else if (struct_info.fields.len <= std.math.maxInt(u32)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_dict_4)); + try writeInt(u32, @intCast(struct_info.fields.len), writer); + } + + inline for (struct_info.fields) |field| { + try innerSerialize([]const u8, field.name, writer); + try innerSerialize(field.type, @field(value, field.name), writer); + } + } + }, + else => @compileError("Unsupported type"), + } +} + +fn parseArrayOrSlice(comptime T: type, value: []const T, writer: *std.Io.Writer) !void { + switch (@typeInfo(T)) { + .int => |int| { + try writeVectorMarker(value, writer); + try writer.writeByte(getIntMarker(int)); + for (value) |val| { + try writeInt(std.math.ByteAlignedInt(T), val, writer); + } + }, + .float => |float| { + try writeVectorMarker(value, writer); + try writer.writeByte(getFloatMarker(float)); + for (value) |val| { + try writeFloat(std.math.ByteAlignedInt(T), val, writer); + } + }, + .bool => { + try writeVectorMarker(value, writer); + for (value) |val| { + try writer.writeByte(0x01 + @as(u8, @intFromBool(val))); + } + }, + else => { + if (value.len <= 15) { + try writer.writeByte(@intFromEnum(MarkerType.empty_tuple) + @as(u8, @intCast(value.len))); + } else if (value.len <= std.math.maxInt(u8)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_1)); + try writeInt(u8, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u16)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_2)); + try writeInt(u16, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u32)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_tuple_4)); + try writeInt(u32, @intCast(value.len), writer); + } + + for (value) |val| { + try innerSerialize(T, val, writer); + } + }, + } +} + +fn writeInt(comptime T: type, value: T, writer: *std.Io.Writer) !void { + try writer.writeAll(&@as([@divExact(@typeInfo(T).int.bits, 8)]u8, @bitCast(std.mem.nativeToLittle(T, @intCast(value))))); +} + +fn writeFloat(comptime T: type, value: T, writer: *std.Io.Writer) !void { + try writer.writeAll(&@as([@divExact(@typeInfo(T).float.bits, 8)]u8, @bitCast(std.mem.nativeToLittle(T, value)))); +} + +fn writeVectorMarker(value: anytype, writer: *std.Io.Writer) !void { + if (value.len <= 15) { + try writer.writeByte(@intFromEnum(MarkerType.empty_vector) + value.len); + } else if (value.len <= std.math.maxInt(u8)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_vector_1)); + try writeInt(u8, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u16)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_vector_2)); + try writeInt(u16, @intCast(value.len), writer); + } else if (value.len <= std.math.maxInt(u32)) { + try writer.writeByte(@intFromEnum(MarkerType.arbitrary_vector_4)); + try writeInt(u32, @intCast(value.len), writer); + } +} + +fn getIntMarker(int: std.builtin.Type.Int) u8 { + const base = @intFromEnum(if (int.signedness == .unsigned) MarkerType.unsigned_int_8 else MarkerType.signed_int_8); + + return switch (int.bits) { + 8 => base, + 16 => base + 1, + 32 => base + 2, + 64 => base + 3, + 128 => base + 4, + 256 => base + 5, + 512 => base + 6, + else => base + 0x0F, + }; +} + +fn getFloatMarker(float: std.builtin.Type.Float) u8 { + return switch (float.bits) { + 16 => @intFromEnum(MarkerType.half_float), + 32 => @intFromEnum(MarkerType.single_float), + 64 => @intFromEnum(MarkerType.double_float), + 80 => @intFromEnum(MarkerType.extended_float_80), + 128 => @intFromEnum(MarkerType.quadruple_float), + else => unreachable, + }; +} diff --git a/zig/src/static.zig b/zig/src/static.zig new file mode 100644 index 0000000..adf9ade --- /dev/null +++ b/zig/src/static.zig @@ -0,0 +1,308 @@ +const std = @import("std"); +const Scanner = @import("Scanner.zig"); + +const assert = std.debug.assert; + +pub const ParseOptions = struct { + /// Wether arrays should be allowed to be partially filled. + /// + /// This only affects arrays with optionals meaning + /// `[50]?u8` can be `[30]u8 ++ [20]null` + /// + /// The default behavior (false) checks if the HBP payload has the exact same length as the array. + allow_empty_array_elements: bool = false, + /// Allow parsing `i8` as `u8` and vice-versa. + ignore_integer_signedness: bool = false, + /// Use `std.enums.fromInt` instead of attempting to cast. + safe_enum_parsing: bool = false, + float_behavior: enum(u1) { + widen, + preserve, + } = .preserve, + /// Wether to try parsing types that do not start with `0xF0` (optional marker). + non_typed_optionals: enum(u1) { + @"error", + allow, + } = .@"error", +}; + +fn ParseOutputType(comptime T: type) type { + return switch (@typeInfo(T)) { + .int => alignIntegerType(T), + else => T, + }; +} + +pub fn parseFromSlice(comptime T: type, slice: []const u8, gpa: std.mem.Allocator, comptime options: ParseOptions) !ParseOutputType(T) { + var scanner: Scanner = .init(slice); + defer scanner.deinit(); + + return parseFromTokenSource(T, &scanner, gpa, options); +} + +pub fn parseFromTokenSource(comptime T: type, scanner: *Scanner, gpa: std.mem.Allocator, comptime options: ParseOptions) !ParseOutputType(T) { + assert(try scanner.next() == .identifier); + const value = try innerParse(T, scanner, gpa, options); + assert(try scanner.next() == .eos); + return value; +} + +/// Allocator is only used for dynamic slices and strings +pub fn innerParse(comptime T: type, scanner: *Scanner, gpa: std.mem.Allocator, comptime options: ParseOptions) !ParseOutputType(T) { + switch (@typeInfo(T)) { + .void => return, + .null => { + return switch (try scanner.next()) { + .null => null, + else => error.UnexpectedToken, + }; + }, + .bool => { + return switch (try scanner.next()) { + .false => false, + .true => true, + else => error.UnexpectedToken, + }; + }, + .int => |int| { + const token = try scanner.next(); + if (token != .int) return error.UnexpectedToken; + if (comptime !options.ignore_integer_signedness) { + if (token.int.signedness != int.signedness) return error.WrongIntegerType; + } + + return sliceToInt(T, token.int.view); + }, + .float => |float| { + const token = try scanner.next(); + if (token != .float) return error.UnexpectedToken; + if (comptime options.float_behavior != .widen) { + if (float.bits != token.float.bits) return error.CannotWidenFloat; + } + assert(float.bits >= token.float.bits); + + return @as(T, switch (token.float.bits) { + 16 => std.mem.bytesToValue(f16, token.float.view), + 32 => std.mem.bytesToValue(f32, token.float.view), + 64 => std.mem.bytesToValue(f64, token.float.view), + 80 => std.mem.bytesToValue(f80, token.float.view), + 128 => std.mem.bytesToValue(f128, token.float.view), + else => unreachable, + }); + }, + .optional => |optional| { + switch (try scanner.peekNextTokenType()) { + .optional => { + _ = try scanner.next(); + if (try scanner.peekNextTokenType() == .null) { + _ = try scanner.next(); + return null; + } + + return try innerParse(optional.child, scanner, gpa, options); + }, + else => return if (comptime options.non_typed_optionals == .allow) try innerParse(optional.child, scanner, gpa, options) else error.UnexpectedToken, + } + }, + .@"enum" => |enum_info| { + if (try scanner.next() != .@"enum") return error.UnexpectedToken; + const token = try scanner.next(); + if (token != .int) return error.UnexpectedToken; + + if (comptime options.safe_enum_parsing) { + return std.enums.fromInt(T, sliceToInt(enum_info.tag_type, token.int.view)) orelse error.InvalidEnumTag; + } else { + return @enumFromInt(sliceToInt(enum_info.tag_type, token.int.view)); + } + }, + .@"union" => |union_info| { + if (union_info.tag_type) |tag_type| { + const token = try scanner.next(); + if (token != .@"union") return error.UnexpectedToken; + const union_tag = try innerParse(@typeInfo(tag_type).@"enum".tag_type, scanner, gpa, options); + inline for (union_info.fields) |field| { + if (std.mem.eql(u8, field.name, @tagName(@as(tag_type, @enumFromInt(union_tag))))) return @unionInit(T, field.name, try innerParse(field.type, scanner, gpa, options)); + } + } else @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'"); + + return error.InvalidUnion; + }, + .array => |array_info| { + switch (@typeInfo(array_info.child)) { + else => |t| { + const token = try scanner.next(); + if (token != .tuple) return error.UnexpectedToken; + + if (array_info.len != token.tuple) { + if (!options.allow_empty_array_elements) + return error.InvalidArrayComponent + else if (t != .optional) + return error.PartialOptionalsOnly; + } + var arr: [array_info.len]array_info.child = if (t == .optional) @splat(null) else undefined; + + for (0..token.tuple) |i| { + arr[i] = try innerParse(array_info.child, scanner, gpa, options); + } + + return arr; + }, + } + }, + .pointer => |pointer_info| { + switch (pointer_info.size) { + .slice => { + if (pointer_info.is_const and pointer_info.child == u8) { + const token = try scanner.next(); + if (token != .string) return error.UnexpectedToken; + + const str = try gpa.dupe(u8, scanner.input[scanner.cursor .. scanner.cursor + token.string]); + scanner.cursor += token.string; + scanner.state = .post_value; + return str; + } + + switch (@typeInfo(pointer_info.child)) { + .int => |int| { + const token = try scanner.next(); + if (token != .vector) return error.UnexpectedToken; + + const first = try scanner.next(); + if (first != .int) return error.UnexpectedToken; + + if (comptime !options.ignore_integer_signedness) { + if (first.int.signedness != int.signedness) return error.WrongIntegerType; + } + + const element_byte_length = first.int.view.len; + const N = alignIntegerType(pointer_info.child); + var arr: std.ArrayList(N) = try .initCapacity(gpa, 1); + + try arr.append(gpa, sliceToInt(N, first.int.view)); + + // The first element is already retrieved + for (1..token.vector) |_| { + const value_start = scanner.cursor; + scanner.cursor += element_byte_length; + try arr.append(gpa, sliceToInt(N, scanner.input[value_start..scanner.cursor])); + } + + return try arr.toOwnedSlice(gpa); + }, + .float => { + const token = try scanner.next(); + if (token != .vector) return error.UnexpectedToken; + + const first = try scanner.next(); + if (first != .float) return error.UnexpectedToken; + + const element_byte_length = first.float.view.len; + const N = alignIntegerType(pointer_info.child); + var arr: std.ArrayList(N) = try .initCapacity(gpa, 1); + + try arr.append(gpa, sliceToInt(N, first.float.view)); + + // The first element is already retrieved + for (1..token.vector) |_| { + const value_start = scanner.cursor; + scanner.cursor += element_byte_length; + try arr.append(gpa, sliceToInt(N, scanner.input[value_start..scanner.cursor])); + } + return try arr.toOwnedSlice(gpa); + }, + .bool => { + const token = try scanner.next(); + if (token != .vector) return error.UnexpectedToken; + var arr: std.ArrayList(bool) = .empty; + + for (0..token.tuple) |_| { + const b = try scanner.next(); + if (b != .bool) return error.UnexpectedToken; + try arr.append(gpa, switch (b.bool) { + .false => false, + .true => true, + }); + } + + return try arr.toOwnedSlice(gpa); + }, + else => { + const token = try scanner.next(); + if (token != .tuple) return error.UnexpectedToken; + var arr: std.ArrayList(pointer_info.child) = .empty; + + for (0..token.tuple) |_| { + try arr.append(gpa, try innerParse(pointer_info.child, scanner, gpa, options)); + } + + return try arr.toOwnedSlice(gpa); + }, + } + }, + else => @compileError("Unsupported pointer type"), + } + }, + .@"struct" => |struct_info| { + if (struct_info.layout == .@"packed") { + const token = try scanner.peekNextTokenType(); + if (token != .int) return error.UnexpectedToken; + return @bitCast(try innerParse(struct_info.backing_integer.?, scanner, gpa, options)); + } + + if (struct_info.is_tuple) { + const token = try scanner.next(); + if (token != .tuple) return error.UnexpectedToken; + assert(struct_info.fields.len == token.tuple); + + var r: T = undefined; + + inline for (struct_info.fields, 0..) |field, i| { + r[i] = try innerParse(field.type, scanner, gpa, options); + } + + return r; + } + + const token = try scanner.next(); + if (token != .@"struct") return error.UnexpectedToken; + assert(struct_info.fields.len == token.@"struct"); + var r: T = undefined; + + for (0..token.@"struct") |_| { + const field_name = try innerParse([]const u8, scanner, gpa, options); + defer gpa.free(field_name); + inline for (struct_info.fields) |field| { + if (std.mem.eql(u8, field.name, field_name)) { + @field(r, field.name) = try innerParse(field.type, scanner, gpa, options); + } + } + } + + return r; + }, + .comptime_int, .comptime_float => error.IncompatibleTypes, + else => return error.TODO, + } +} + +inline fn alignIntegerType(comptime T: type) type { + const int = @typeInfo(T).int; + // 0 bit integers mostly happen when using enums with 1 single element + // We require every type to be at least 1 byte long to be parsed + if (int.bits == 0) return std.meta.Int(int.signedness, 8); + return std.math.ByteAlignedInt(T); +} + +fn sliceToInt(comptime T: type, slice: []const u8) alignIntegerType(T) { + const N = alignIntegerType(T); + const byte_length = @divExact(@typeInfo(N).int.bits, 8); + + if (slice.len < byte_length) { + var buf = std.mem.zeroes([byte_length]u8); + @memcpy(buf[0..slice.len], slice); + return std.mem.readInt(N, &buf, .little); + } + + assert(slice.len == byte_length); + return std.mem.readInt(N, slice[0..byte_length], .little); +}