diff --git a/CLAUDE.md b/CLAUDE.md index 64f8ad5c4..ff1a491f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,18 +60,19 @@ Trunk-based. PRs fine but always squash or rebase — no merge commits. Keep com ./bench JavaVsJniReadBenchmark.javaReadVolume # benchmark — always ClassName.methodName filter ``` -Regenerate after editing `.fbs`/`.proto`: +Regenerate after editing `.fbs`/`.proto` (both generators are in-house, no external tools): ```bash -brew install flatbuffers # only for .fbs edits (any flatc version; guard auto-stripped) -./mvnw compile -pl proto-gen # only on .proto edits +./mvnw compile -pl fbs-gen,proto-gen # build the generators ./mvnw generate-sources -pl core -P regenerate-sources # then commit ``` -`flatc` runs whenever the profile is active; if you only changed `.proto`, revert spurious -`fbs/` diffs: `git checkout -- core/src/main/java/io/github/dfa1/vortex/fbs/`. Proto-to-Java -is in-process via `proto-gen` (no `protoc`/`protobuf-java`): one record per message with static -`decode(MemorySegment, long, long)` + `encode()` operating directly on a segment. +Both schema languages are compiled in-process to MemorySegment-native Java, with no +`flatc`/`protoc` and no `com.google.flatbuffers`/`protobuf-java` runtime (ADR 0017): +- **`.fbs` → `fbs-gen`** (`io.github.dfa1.vortex.fbsgen`): generates readers extending + `FbsTable`/`FbsStruct` and builders over `FbsBuilder` (all in `io.github.dfa1.vortex.fbsrt`). +- **`.proto` → `proto-gen`**: one record per message with static `decode(MemorySegment, long, + long)` + `encode()` operating directly on a segment. ### Mutation testing diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java index 8205ed3e3..e750c2981 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java @@ -191,8 +191,9 @@ private void indexStatsChildren(InspectorTree.Node node) { } private static boolean hasLeadingStats(Layout layout) { - java.nio.ByteBuffer meta = layout.metadata(); - return meta != null && meta.hasRemaining() && meta.get(meta.position()) == 1; + MemorySegment meta = layout.metadata(); + return meta != null && meta.byteSize() > 0 + && meta.get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0) == 1; } private void prefetchTopColumns() { @@ -671,8 +672,7 @@ private void runStatsLoad(InspectorTree.Node anchor) { statsCache.put(anchor, new DataState.Failed("no column dtype")); return; } - DType.Struct statsDtype = ZonedStatsSchema.statsTableDtype( - columnDtype, anchorLayout.metadata()); + DType.Struct statsDtype = ZonedStatsSchema.statsTableDtype(columnDtype, anchorLayout.metadata()); if (statsDtype.fieldNames().isEmpty()) { statsCache.put(anchor, new DataState.Failed("no stats present in metadata")); return; diff --git a/core/pom.xml b/core/pom.xml index 11a2c3fd5..a80b74c58 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -15,11 +15,6 @@ - - - com.google.flatbuffers - flatbuffers-java - org.junit.jupiter @@ -59,53 +54,51 @@ regenerate-sources - + org.codehaus.mojo exec-maven-plugin - flatc-generate + fbsgen-generate generate-sources exec - flatc - ${project.basedir}/src/main/flatbuffers + java - --java - -I - . - -o - ${project.basedir}/src/main/java - vortex-file/footer.fbs - vortex-layout/layout.fbs - vortex-array/array.fbs - vortex-dtype/dtype.fbs + -cp + ${project.basedir}/../fbs-gen/target/classes + io.github.dfa1.vortex.fbsgen.Main + --out + ${project.basedir}/src/main/java/io/github/dfa1/vortex/fbs + ${project.basedir}/src/main/flatbuffers/vortex-array/array.fbs + ${project.basedir}/src/main/flatbuffers/vortex-dtype/dtype.fbs + ${project.basedir}/src/main/flatbuffers/vortex-file/footer.fbs + ${project.basedir}/src/main/flatbuffers/vortex-layout/layout.fbs - protogen-generate generate-sources @@ -128,51 +121,6 @@ - - - - org.apache.maven.plugins - maven-antrun-plugin - - - flatc-strip-version-guard - generate-sources - - run - - - - - - - - - - - diff --git a/core/src/main/java/io/github/dfa1/vortex/core/DType.java b/core/src/main/java/io/github/dfa1/vortex/core/DType.java index d62db40f7..b187cc5e0 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/DType.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/DType.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.core; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -267,7 +267,7 @@ record FixedSizeList(DType elementType, int fixedSize, boolean nullable) impleme record Extension( String extensionId, DType storageDType, - ByteBuffer metadata, + MemorySegment metadata, boolean nullable ) implements DType { @@ -280,11 +280,52 @@ record Extension( /// @throws VortexException if `metadata` carries more than /// [#MAX_METADATA_SIZE] readable bytes public Extension { - if (metadata != null && metadata.remaining() > MAX_METADATA_SIZE) { + if (metadata != null && metadata.byteSize() > MAX_METADATA_SIZE) { throw new VortexException("extension metadata too large: " - + metadata.remaining() + " > " + MAX_METADATA_SIZE); + + metadata.byteSize() + " > " + MAX_METADATA_SIZE); } } + + /// Value equality with content-based metadata comparison. + /// + /// [MemorySegment] uses identity equality, so the default record `equals` would + /// treat a heap-built dtype and the same dtype read back from a mapped file as + /// unequal. Extension is a value type (compared in schema round-trips and as a + /// component of [Struct]/[List] dtypes), so metadata is compared by bytes — + /// matching the content semantics the previous `ByteBuffer` component had. + /// + /// @param o other object + /// @return whether `o` is an Extension with equal id, storage, nullability and metadata bytes + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Extension other)) { + return false; + } + return nullable == other.nullable + && extensionId.equals(other.extensionId) + && storageDType.equals(other.storageDType) + && metadataEquals(metadata, other.metadata); + } + + /// @return a hash consistent with [#equals(Object)] (metadata hashed by content) + @Override + public int hashCode() { + int h = java.util.Objects.hash(extensionId, storageDType, nullable); + if (metadata != null) { + h = 31 * h + java.util.Arrays.hashCode(metadata.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE)); + } + return h; + } + + private static boolean metadataEquals(MemorySegment a, MemorySegment b) { + if (a == null || b == null) { + return a == b; + } + return a.byteSize() == b.byteSize() && a.mismatch(b) == -1; + } } /// Variant logical type for semi-structured data (analogous to Parquet variant / JSON). diff --git a/core/src/main/java/io/github/dfa1/vortex/extension/TimeDtype.java b/core/src/main/java/io/github/dfa1/vortex/extension/TimeDtype.java index 013350314..bc42219c2 100644 --- a/core/src/main/java/io/github/dfa1/vortex/extension/TimeDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/extension/TimeDtype.java @@ -5,7 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.TimeUnit; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; /// Static factories and metadata accessors for `vortex.time` extension dtypes. /// @@ -36,8 +37,7 @@ public static DType.Extension of(TimeUnit unit, boolean nullable) { case Microseconds, Nanoseconds -> PType.I64; case Days -> throw new IllegalArgumentException("Days unit not valid for vortex.time"); }; - ByteBuffer meta = ByteBuffer.allocate(1); - meta.put(0, (byte) unit.ordinal()); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) unit.ordinal()}); return new DType.Extension( ExtensionId.VORTEX_TIME.id(), new DType.Primitive(storage, nullable), @@ -51,10 +51,10 @@ public static DType.Extension of(TimeUnit unit, boolean nullable) { /// @return the recorded time unit /// @throws VortexException if metadata is missing or empty public static TimeUnit readUnit(DType.Extension ext) { - ByteBuffer meta = ext.metadata(); - if (meta == null || !meta.hasRemaining()) { + MemorySegment meta = ext.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException("missing TimeUnit metadata byte for " + ext.extensionId()); } - return TimeUnit.fromTag(meta.get(meta.position())); + return TimeUnit.fromTag(meta.get(ValueLayout.JAVA_BYTE, 0)); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/extension/TimestampDtype.java b/core/src/main/java/io/github/dfa1/vortex/extension/TimestampDtype.java index 4fa9d3e91..47489c16a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/extension/TimestampDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/extension/TimestampDtype.java @@ -5,8 +5,10 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.TimeUnit; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.Optional; @@ -37,12 +39,10 @@ public static DType.Extension of(boolean nullable) { /// @return matching extension dtype public static DType.Extension of(TimeUnit unit, ZoneId zone, boolean nullable) { byte[] tzBytes = zone == null ? new byte[0] : zone.getId().getBytes(StandardCharsets.UTF_8); - ByteBuffer meta = ByteBuffer.allocate(3 + tzBytes.length).order(ByteOrder.LITTLE_ENDIAN); - meta.put(0, (byte) unit.ordinal()); - meta.putShort(1, (short) tzBytes.length); - for (int k = 0; k < tzBytes.length; k++) { - meta.put(3 + k, tzBytes[k]); - } + MemorySegment meta = MemorySegment.ofArray(new byte[3 + tzBytes.length]); + meta.set(ValueLayout.JAVA_BYTE, 0, (byte) unit.ordinal()); + meta.set(LE_SHORT, 1, (short) tzBytes.length); + MemorySegment.copy(tzBytes, 0, meta, ValueLayout.JAVA_BYTE, 3, tzBytes.length); return new DType.Extension( ExtensionId.VORTEX_TIMESTAMP.id(), new DType.Primitive(PType.I64, nullable), @@ -56,11 +56,11 @@ public static DType.Extension of(TimeUnit unit, ZoneId zone, boolean nullable) { /// @return the recorded time unit /// @throws VortexException if metadata is missing or empty public static TimeUnit readUnit(DType.Extension ext) { - ByteBuffer meta = ext.metadata(); - if (meta == null || !meta.hasRemaining()) { + MemorySegment meta = ext.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException("missing TimeUnit metadata byte for " + ext.extensionId()); } - return TimeUnit.fromTag(meta.get(meta.position())); + return TimeUnit.fromTag(meta.get(ValueLayout.JAVA_BYTE, 0)); } /// Reads the optional IANA timezone from the metadata's UTF-8 suffix. @@ -69,25 +69,20 @@ public static TimeUnit readUnit(DType.Extension ext) { /// @return parsed zone id, or empty when no timezone is recorded /// @throws VortexException if the metadata is truncated mid-string public static Optional timezone(DType.Extension ext) { - ByteBuffer meta = ext.metadata(); - if (meta == null || meta.remaining() < 3) { + MemorySegment meta = ext.metadata(); + if (meta == null || meta.byteSize() < 3) { return Optional.empty(); } - ByteBuffer le = meta.duplicate().order(ByteOrder.LITTLE_ENDIAN); - int basePos = le.position(); - int tzLen = Short.toUnsignedInt(le.getShort(basePos + 1)); + int tzLen = Short.toUnsignedInt(meta.get(LE_SHORT, 1)); if (tzLen == 0) { return Optional.empty(); } - if (le.remaining() < 3 + tzLen) { + if (meta.byteSize() < 3 + tzLen) { throw new VortexException( "timestamp metadata truncated: declared tz_len=" - + tzLen + " but only " + (le.remaining() - 3) + " bytes available"); - } - byte[] tzBytes = new byte[tzLen]; - for (int k = 0; k < tzLen; k++) { - tzBytes[k] = le.get(basePos + 3 + k); + + tzLen + " but only " + (meta.byteSize() - 3) + " bytes available"); } + byte[] tzBytes = meta.asSlice(3, tzLen).toArray(ValueLayout.JAVA_BYTE); return Optional.of(ZoneId.of(new String(tzBytes, StandardCharsets.UTF_8))); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Array.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Array.java deleted file mode 100644 index 525d11bdc..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Array.java +++ /dev/null @@ -1,122 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Array extends com.google.flatbuffers.Table { - public static Array getRootAsArray(ByteBuffer _bb) { - return getRootAsArray(_bb, new Array()); - } - - public static Array getRootAsArray(ByteBuffer _bb, Array obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createArray(FlatBufferBuilder builder, - int rootOffset, - int buffersOffset) { - builder.startTable(2); - Array.addBuffers(builder, buffersOffset); - Array.addRoot(builder, rootOffset); - return Array.endArray(builder); - } - - public static void startArray(FlatBufferBuilder builder) { - builder.startTable(2); - } - - public static void addRoot(FlatBufferBuilder builder, int rootOffset) { - builder.addOffset(0, rootOffset, 0); - } - - public static void addBuffers(FlatBufferBuilder builder, int buffersOffset) { - builder.addOffset(1, buffersOffset, 0); - } - - public static void startBuffersVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(8, numElems, 4); - } - - public static int endArray(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public static void finishArrayBuffer(FlatBufferBuilder builder, int offset) { - builder.finish(offset); - } - - public static void finishSizePrefixedArrayBuffer(FlatBufferBuilder builder, int offset) { - builder.finishSizePrefixed(offset); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Array __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * The array's hierarchical definition. - */ - public io.github.dfa1.vortex.fbs.ArrayNode root() { - return root(new io.github.dfa1.vortex.fbs.ArrayNode()); - } - - public io.github.dfa1.vortex.fbs.ArrayNode root(io.github.dfa1.vortex.fbs.ArrayNode obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - /** - * The locations of the data buffers of the array - */ - public io.github.dfa1.vortex.fbs.Buffer buffers(int j) { - return buffers(new io.github.dfa1.vortex.fbs.Buffer(), j); - } - - public io.github.dfa1.vortex.fbs.Buffer buffers(io.github.dfa1.vortex.fbs.Buffer obj, int j) { - int o = __offset(6); - return o != 0 ? obj.__assign(__vector(o) + j * 8, bb) : null; - } - - public int buffersLength() { - int o = __offset(6); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.Buffer.Vector buffersVector() { - return buffersVector(new io.github.dfa1.vortex.fbs.Buffer.Vector()); - } - - public io.github.dfa1.vortex.fbs.Buffer.Vector buffersVector(io.github.dfa1.vortex.fbs.Buffer.Vector obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__vector(o), 8, bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Array get(int j) { - return get(new Array(), j); - } - - public Array get(Array obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayNode.java b/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayNode.java deleted file mode 100644 index 825851fe5..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayNode.java +++ /dev/null @@ -1,215 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.ByteVector; -import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.ShortVector; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class ArrayNode extends com.google.flatbuffers.Table { - public static ArrayNode getRootAsArrayNode(ByteBuffer _bb) { - return getRootAsArrayNode(_bb, new ArrayNode()); - } - - public static ArrayNode getRootAsArrayNode(ByteBuffer _bb, ArrayNode obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createArrayNode(FlatBufferBuilder builder, - int encoding, - int metadataOffset, - int childrenOffset, - int buffersOffset, - int statsOffset) { - builder.startTable(5); - ArrayNode.addStats(builder, statsOffset); - ArrayNode.addBuffers(builder, buffersOffset); - ArrayNode.addChildren(builder, childrenOffset); - ArrayNode.addMetadata(builder, metadataOffset); - ArrayNode.addEncoding(builder, encoding); - return ArrayNode.endArrayNode(builder); - } - - public static void startArrayNode(FlatBufferBuilder builder) { - builder.startTable(5); - } - - public static void addEncoding(FlatBufferBuilder builder, int encoding) { - builder.addShort(0, (short) encoding, (short) 0); - } - - public static void addMetadata(FlatBufferBuilder builder, int metadataOffset) { - builder.addOffset(1, metadataOffset, 0); - } - - public static int createMetadataVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createMetadataVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startMetadataVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static void addChildren(FlatBufferBuilder builder, int childrenOffset) { - builder.addOffset(2, childrenOffset, 0); - } - - public static int createChildrenVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startChildrenVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addBuffers(FlatBufferBuilder builder, int buffersOffset) { - builder.addOffset(3, buffersOffset, 0); - } - - public static int createBuffersVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(2, data.length, 2); - for (int i = data.length - 1; i >= 0; i--) builder.addShort((short) data[i]); - return builder.endVector(); - } - - public static void startBuffersVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(2, numElems, 2); - } - - public static void addStats(FlatBufferBuilder builder, int statsOffset) { - builder.addOffset(4, statsOffset, 0); - } - - public static int endArrayNode(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public ArrayNode __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public int encoding() { - int o = __offset(4); - return o != 0 ? bb.getShort(o + bb_pos) & 0xFFFF : 0; - } - - public int metadata(int j) { - int o = __offset(6); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int metadataLength() { - int o = __offset(6); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector metadataVector() { - return metadataVector(new ByteVector()); - } - - public ByteVector metadataVector(ByteVector obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer metadataAsByteBuffer() { - return __vector_as_bytebuffer(6, 1); - } - - public ByteBuffer metadataInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 6, 1); - } - - public io.github.dfa1.vortex.fbs.ArrayNode children(int j) { - return children(new io.github.dfa1.vortex.fbs.ArrayNode(), j); - } - - public io.github.dfa1.vortex.fbs.ArrayNode children(io.github.dfa1.vortex.fbs.ArrayNode obj, int j) { - int o = __offset(8); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int childrenLength() { - int o = __offset(8); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.ArrayNode.Vector childrenVector() { - return childrenVector(new io.github.dfa1.vortex.fbs.ArrayNode.Vector()); - } - - public io.github.dfa1.vortex.fbs.ArrayNode.Vector childrenVector(io.github.dfa1.vortex.fbs.ArrayNode.Vector obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public int buffers(int j) { - int o = __offset(10); - return o != 0 ? bb.getShort(__vector(o) + j * 2) & 0xFFFF : 0; - } - - public int buffersLength() { - int o = __offset(10); - return o != 0 ? __vector_len(o) : 0; - } - - public ShortVector buffersVector() { - return buffersVector(new ShortVector()); - } - - public ShortVector buffersVector(ShortVector obj) { - int o = __offset(10); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer buffersAsByteBuffer() { - return __vector_as_bytebuffer(10, 2); - } - - public ByteBuffer buffersInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 10, 2); - } - - public io.github.dfa1.vortex.fbs.ArrayStats stats() { - return stats(new io.github.dfa1.vortex.fbs.ArrayStats()); - } - - public io.github.dfa1.vortex.fbs.ArrayStats stats(io.github.dfa1.vortex.fbs.ArrayStats obj) { - int o = __offset(12); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public ArrayNode get(int j) { - return get(new ArrayNode(), j); - } - - public ArrayNode get(ArrayNode obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/ArraySpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/ArraySpec.java deleted file mode 100644 index 47ae98e23..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/ArraySpec.java +++ /dev/null @@ -1,86 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * An `ArraySpec` describes the type of a particular array. - *

- * These are identified by a globally unique string identifier, and looked up in the Vortex registry - * at read-time. - */ -@SuppressWarnings("unused") -public final class ArraySpec extends com.google.flatbuffers.Table { - public static ArraySpec getRootAsArraySpec(ByteBuffer _bb) { - return getRootAsArraySpec(_bb, new ArraySpec()); - } - - public static ArraySpec getRootAsArraySpec(ByteBuffer _bb, ArraySpec obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createArraySpec(FlatBufferBuilder builder, - int idOffset) { - builder.startTable(1); - ArraySpec.addId(builder, idOffset); - return ArraySpec.endArraySpec(builder); - } - - public static void startArraySpec(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addId(FlatBufferBuilder builder, int idOffset) { - builder.addOffset(0, idOffset, 0); - } - - public static int endArraySpec(FlatBufferBuilder builder) { - int o = builder.endTable(); - builder.required(o, 4); // id - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public ArraySpec __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public String id() { - int o = __offset(4); - return o != 0 ? __string(o + bb_pos) : null; - } - - public ByteBuffer idAsByteBuffer() { - return __vector_as_bytebuffer(4, 1); - } - - public ByteBuffer idInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 4, 1); - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public ArraySpec get(int j) { - return get(new ArraySpec(), j); - } - - public ArraySpec get(ArraySpec obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayStats.java deleted file mode 100644 index a9233cbd5..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/ArrayStats.java +++ /dev/null @@ -1,311 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.ByteVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class ArrayStats extends com.google.flatbuffers.Table { - public static ArrayStats getRootAsArrayStats(ByteBuffer _bb) { - return getRootAsArrayStats(_bb, new ArrayStats()); - } - - public static ArrayStats getRootAsArrayStats(ByteBuffer _bb, ArrayStats obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createArrayStats(FlatBufferBuilder builder, - int minOffset, - int minPrecision, - int maxOffset, - int maxPrecision, - int sumOffset, - boolean isSorted, - boolean isStrictSorted, - boolean isConstant, - long nullCount, - long uncompressedSizeInBytes, - long nanCount) { - builder.startTable(11); - ArrayStats.addNanCount(builder, nanCount); - ArrayStats.addUncompressedSizeInBytes(builder, uncompressedSizeInBytes); - ArrayStats.addNullCount(builder, nullCount); - ArrayStats.addSum(builder, sumOffset); - ArrayStats.addMax(builder, maxOffset); - ArrayStats.addMin(builder, minOffset); - ArrayStats.addIsConstant(builder, isConstant); - ArrayStats.addIsStrictSorted(builder, isStrictSorted); - ArrayStats.addIsSorted(builder, isSorted); - ArrayStats.addMaxPrecision(builder, maxPrecision); - ArrayStats.addMinPrecision(builder, minPrecision); - return ArrayStats.endArrayStats(builder); - } - - public static void startArrayStats(FlatBufferBuilder builder) { - builder.startTable(11); - } - - public static void addMin(FlatBufferBuilder builder, int minOffset) { - builder.addOffset(0, minOffset, 0); - } - - public static int createMinVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createMinVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startMinVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static void addMinPrecision(FlatBufferBuilder builder, int minPrecision) { - builder.addByte(1, (byte) minPrecision, (byte) 0); - } - - public static void addMax(FlatBufferBuilder builder, int maxOffset) { - builder.addOffset(2, maxOffset, 0); - } - - public static int createMaxVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createMaxVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startMaxVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static void addMaxPrecision(FlatBufferBuilder builder, int maxPrecision) { - builder.addByte(3, (byte) maxPrecision, (byte) 0); - } - - public static void addSum(FlatBufferBuilder builder, int sumOffset) { - builder.addOffset(4, sumOffset, 0); - } - - public static int createSumVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createSumVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startSumVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static void addIsSorted(FlatBufferBuilder builder, boolean isSorted) { - builder.addBoolean(5, isSorted, false); - } - - public static void addIsStrictSorted(FlatBufferBuilder builder, boolean isStrictSorted) { - builder.addBoolean(6, isStrictSorted, false); - } - - public static void addIsConstant(FlatBufferBuilder builder, boolean isConstant) { - builder.addBoolean(7, isConstant, false); - } - - public static void addNullCount(FlatBufferBuilder builder, long nullCount) { - builder.addLong(8, nullCount, 0L); - } - - public static void addUncompressedSizeInBytes(FlatBufferBuilder builder, long uncompressedSizeInBytes) { - builder.addLong(9, uncompressedSizeInBytes, 0L); - } - - public static void addNanCount(FlatBufferBuilder builder, long nanCount) { - builder.addLong(10, nanCount, 0L); - } - - public static int endArrayStats(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public ArrayStats __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * Protobuf serialized ScalarValue - */ - public int min(int j) { - int o = __offset(4); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int minLength() { - int o = __offset(4); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector minVector() { - return minVector(new ByteVector()); - } - - public ByteVector minVector(ByteVector obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer minAsByteBuffer() { - return __vector_as_bytebuffer(4, 1); - } - - public ByteBuffer minInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 4, 1); - } - - public int minPrecision() { - int o = __offset(6); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public int max(int j) { - int o = __offset(8); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int maxLength() { - int o = __offset(8); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector maxVector() { - return maxVector(new ByteVector()); - } - - public ByteVector maxVector(ByteVector obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer maxAsByteBuffer() { - return __vector_as_bytebuffer(8, 1); - } - - public ByteBuffer maxInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 8, 1); - } - - public int maxPrecision() { - int o = __offset(10); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public int sum(int j) { - int o = __offset(12); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int sumLength() { - int o = __offset(12); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector sumVector() { - return sumVector(new ByteVector()); - } - - public ByteVector sumVector(ByteVector obj) { - int o = __offset(12); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer sumAsByteBuffer() { - return __vector_as_bytebuffer(12, 1); - } - - public ByteBuffer sumInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 12, 1); - } - - public boolean hasIsSorted() { - return 0 != __offset(14); - } - - public boolean isSorted() { - int o = __offset(14); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public boolean hasIsStrictSorted() { - return 0 != __offset(16); - } - - public boolean isStrictSorted() { - int o = __offset(16); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public boolean hasIsConstant() { - return 0 != __offset(18); - } - - public boolean isConstant() { - int o = __offset(18); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public boolean hasNullCount() { - return 0 != __offset(20); - } - - public long nullCount() { - int o = __offset(20); - return o != 0 ? bb.getLong(o + bb_pos) : 0L; - } - - public boolean hasUncompressedSizeInBytes() { - return 0 != __offset(22); - } - - public long uncompressedSizeInBytes() { - int o = __offset(22); - return o != 0 ? bb.getLong(o + bb_pos) : 0L; - } - - public boolean hasNanCount() { - return 0 != __offset(24); - } - - public long nanCount() { - int o = __offset(24); - return o != 0 ? bb.getLong(o + bb_pos) : 0L; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public ArrayStats get(int j) { - return get(new ArrayStats(), j); - } - - public ArrayStats get(ArrayStats obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Binary.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Binary.java deleted file mode 100644 index a81841120..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Binary.java +++ /dev/null @@ -1,71 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Binary extends com.google.flatbuffers.Table { - public static Binary getRootAsBinary(ByteBuffer _bb) { - return getRootAsBinary(_bb, new Binary()); - } - - public static Binary getRootAsBinary(ByteBuffer _bb, Binary obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createBinary(FlatBufferBuilder builder, - boolean nullable) { - builder.startTable(1); - Binary.addNullable(builder, nullable); - return Binary.endBinary(builder); - } - - public static void startBinary(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(0, nullable, false); - } - - public static int endBinary(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Binary __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public boolean nullable() { - int o = __offset(4); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Binary get(int j) { - return get(new Binary(), j); - } - - public Binary get(Binary obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Bool.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Bool.java deleted file mode 100644 index 203c8f980..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Bool.java +++ /dev/null @@ -1,71 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Bool extends com.google.flatbuffers.Table { - public static Bool getRootAsBool(ByteBuffer _bb) { - return getRootAsBool(_bb, new Bool()); - } - - public static Bool getRootAsBool(ByteBuffer _bb, Bool obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createBool(FlatBufferBuilder builder, - boolean nullable) { - builder.startTable(1); - Bool.addNullable(builder, nullable); - return Bool.endBool(builder); - } - - public static void startBool(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(0, nullable, false); - } - - public static int endBool(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Bool __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public boolean nullable() { - int o = __offset(4); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Bool get(int j) { - return get(new Bool(), j); - } - - public Bool get(Bool obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Buffer.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Buffer.java deleted file mode 100644 index 5f1820e9d..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Buffer.java +++ /dev/null @@ -1,77 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.Struct; - -import java.nio.ByteBuffer; - -/** - * A Buffer describes the location of a data buffer in the byte stream as a packed 64-bit struct. - */ -@SuppressWarnings("unused") -public final class Buffer extends Struct { - public static int createBuffer(FlatBufferBuilder builder, int padding, int alignmentExponent, int compression, long length) { - builder.prep(4, 8); - builder.putInt((int) length); - builder.putByte((byte) compression); - builder.putByte((byte) alignmentExponent); - builder.putShort((short) padding); - return builder.offset(); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Buffer __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * The length of any padding bytes written immediately before the buffer. - */ - public int padding() { - return bb.getShort(bb_pos + 0) & 0xFFFF; - } - - /** - * The minimum alignment of the buffer, stored as an exponent of 2. - */ - public int alignmentExponent() { - return bb.get(bb_pos + 2) & 0xFF; - } - - /** - * The compression algorithm used to compress the buffer. - */ - public int compression() { - return bb.get(bb_pos + 3) & 0xFF; - } - - /** - * The length of the buffer in bytes. - */ - public long length() { - return (long) bb.getInt(bb_pos + 4) & 0xFFFFFFFFL; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Buffer get(int j) { - return get(new Buffer(), j); - } - - public Buffer get(Buffer obj, int j) { - return obj.__assign(__element(j), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Compression.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Compression.java deleted file mode 100644 index 4ed0dd186..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Compression.java +++ /dev/null @@ -1,21 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -/** - * The compression mechanism used to compress the buffer. - */ -@SuppressWarnings("unused") -public final class Compression { - public static final int None = 0; - public static final int LZ4 = 1; - public static final String[] names = {"None", "LZ4",}; - - private Compression() { - } - - public static String name(int e) { - return names[e]; - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionScheme.java b/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionScheme.java deleted file mode 100644 index 52454f816..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionScheme.java +++ /dev/null @@ -1,20 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -@SuppressWarnings("unused") -public final class CompressionScheme { - public static final int None = 0; - public static final int LZ4 = 1; - public static final int ZLib = 2; - public static final int ZStd = 3; - public static final String[] names = {"None", "LZ4", "ZLib", "ZStd",}; - - private CompressionScheme() { - } - - public static String name(int e) { - return names[e]; - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionSpec.java deleted file mode 100644 index db7eadd43..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/CompressionSpec.java +++ /dev/null @@ -1,74 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * Definition of a compression scheme. - */ -@SuppressWarnings("unused") -public final class CompressionSpec extends com.google.flatbuffers.Table { - public static CompressionSpec getRootAsCompressionSpec(ByteBuffer _bb) { - return getRootAsCompressionSpec(_bb, new CompressionSpec()); - } - - public static CompressionSpec getRootAsCompressionSpec(ByteBuffer _bb, CompressionSpec obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createCompressionSpec(FlatBufferBuilder builder, - int scheme) { - builder.startTable(1); - CompressionSpec.addScheme(builder, scheme); - return CompressionSpec.endCompressionSpec(builder); - } - - public static void startCompressionSpec(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addScheme(FlatBufferBuilder builder, int scheme) { - builder.addByte(0, (byte) scheme, (byte) 0); - } - - public static int endCompressionSpec(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public CompressionSpec __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public int scheme() { - int o = __offset(4); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public CompressionSpec get(int j) { - return get(new CompressionSpec(), j); - } - - public CompressionSpec get(CompressionSpec obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/DType.java b/core/src/main/java/io/github/dfa1/vortex/fbs/DType.java deleted file mode 100644 index a0f650f14..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/DType.java +++ /dev/null @@ -1,90 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class DType extends com.google.flatbuffers.Table { - public static DType getRootAsDType(ByteBuffer _bb) { - return getRootAsDType(_bb, new DType()); - } - - public static DType getRootAsDType(ByteBuffer _bb, DType obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createDType(FlatBufferBuilder builder, - byte typeType, - int typeOffset) { - builder.startTable(2); - DType.addType(builder, typeOffset); - DType.addTypeType(builder, typeType); - return DType.endDType(builder); - } - - public static void startDType(FlatBufferBuilder builder) { - builder.startTable(2); - } - - public static void addTypeType(FlatBufferBuilder builder, byte typeType) { - builder.addByte(0, typeType, 0); - } - - public static void addType(FlatBufferBuilder builder, int typeOffset) { - builder.addOffset(1, typeOffset, 0); - } - - public static int endDType(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public static void finishDTypeBuffer(FlatBufferBuilder builder, int offset) { - builder.finish(offset); - } - - public static void finishSizePrefixedDTypeBuffer(FlatBufferBuilder builder, int offset) { - builder.finishSizePrefixed(offset); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public DType __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public byte typeType() { - int o = __offset(4); - return o != 0 ? bb.get(o + bb_pos) : 0; - } - - public com.google.flatbuffers.Table type(com.google.flatbuffers.Table obj) { - int o = __offset(6); - return o != 0 ? __union(obj, o + bb_pos) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public DType get(int j) { - return get(new DType(), j); - } - - public DType get(DType obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Decimal.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Decimal.java deleted file mode 100644 index 8a73026c9..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Decimal.java +++ /dev/null @@ -1,93 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Decimal extends com.google.flatbuffers.Table { - public static Decimal getRootAsDecimal(ByteBuffer _bb) { - return getRootAsDecimal(_bb, new Decimal()); - } - - public static Decimal getRootAsDecimal(ByteBuffer _bb, Decimal obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createDecimal(FlatBufferBuilder builder, - int precision, - byte scale, - boolean nullable) { - builder.startTable(3); - Decimal.addNullable(builder, nullable); - Decimal.addScale(builder, scale); - Decimal.addPrecision(builder, precision); - return Decimal.endDecimal(builder); - } - - public static void startDecimal(FlatBufferBuilder builder) { - builder.startTable(3); - } - - public static void addPrecision(FlatBufferBuilder builder, int precision) { - builder.addByte(0, (byte) precision, (byte) 0); - } - - public static void addScale(FlatBufferBuilder builder, byte scale) { - builder.addByte(1, scale, 0); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(2, nullable, false); - } - - public static int endDecimal(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Decimal __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public int precision() { - int o = __offset(4); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public byte scale() { - int o = __offset(6); - return o != 0 ? bb.get(o + bb_pos) : 0; - } - - public boolean nullable() { - int o = __offset(8); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Decimal get(int j) { - return get(new Decimal(), j); - } - - public Decimal get(Decimal obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/EncryptionSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/EncryptionSpec.java deleted file mode 100644 index 2674093ca..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/EncryptionSpec.java +++ /dev/null @@ -1,55 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class EncryptionSpec extends com.google.flatbuffers.Table { - public static EncryptionSpec getRootAsEncryptionSpec(ByteBuffer _bb) { - return getRootAsEncryptionSpec(_bb, new EncryptionSpec()); - } - - public static EncryptionSpec getRootAsEncryptionSpec(ByteBuffer _bb, EncryptionSpec obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static void startEncryptionSpec(FlatBufferBuilder builder) { - builder.startTable(0); - } - - public static int endEncryptionSpec(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public EncryptionSpec __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public EncryptionSpec get(int j) { - return get(new EncryptionSpec(), j); - } - - public EncryptionSpec get(EncryptionSpec obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Extension.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Extension.java deleted file mode 100644 index 36ae1d3f5..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Extension.java +++ /dev/null @@ -1,140 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.ByteVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Extension extends com.google.flatbuffers.Table { - public static Extension getRootAsExtension(ByteBuffer _bb) { - return getRootAsExtension(_bb, new Extension()); - } - - public static Extension getRootAsExtension(ByteBuffer _bb, Extension obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createExtension(FlatBufferBuilder builder, - int idOffset, - int storageDtypeOffset, - int metadataOffset) { - builder.startTable(3); - Extension.addMetadata(builder, metadataOffset); - Extension.addStorageDtype(builder, storageDtypeOffset); - Extension.addId(builder, idOffset); - return Extension.endExtension(builder); - } - - public static void startExtension(FlatBufferBuilder builder) { - builder.startTable(3); - } - - public static void addId(FlatBufferBuilder builder, int idOffset) { - builder.addOffset(0, idOffset, 0); - } - - public static void addStorageDtype(FlatBufferBuilder builder, int storageDtypeOffset) { - builder.addOffset(1, storageDtypeOffset, 0); - } - - public static void addMetadata(FlatBufferBuilder builder, int metadataOffset) { - builder.addOffset(2, metadataOffset, 0); - } - - public static int createMetadataVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createMetadataVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startMetadataVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static int endExtension(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Extension __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public String id() { - int o = __offset(4); - return o != 0 ? __string(o + bb_pos) : null; - } - - public ByteBuffer idAsByteBuffer() { - return __vector_as_bytebuffer(4, 1); - } - - public ByteBuffer idInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 4, 1); - } - - public io.github.dfa1.vortex.fbs.DType storageDtype() { - return storageDtype(new io.github.dfa1.vortex.fbs.DType()); - } - - public io.github.dfa1.vortex.fbs.DType storageDtype(io.github.dfa1.vortex.fbs.DType obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public int metadata(int j) { - int o = __offset(8); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int metadataLength() { - int o = __offset(8); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector metadataVector() { - return metadataVector(new ByteVector()); - } - - public ByteVector metadataVector(ByteVector obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer metadataAsByteBuffer() { - return __vector_as_bytebuffer(8, 1); - } - - public ByteBuffer metadataInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 8, 1); - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Extension get(int j) { - return get(new Extension(), j); - } - - public Extension get(Extension obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArray.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArray.java new file mode 100644 index 000000000..1f08f5fb1 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArray.java @@ -0,0 +1,101 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Array` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsArray extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsArray getRootAsFbsArray(MemorySegment seg) { + return new FbsArray().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsArray assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `root` child, or null if absent + public FbsArrayNode root() { + int o = fieldOffset(4); + return o != 0 ? new FbsArrayNode().assign(seg, indirect(pos + o)) : null; + } + + /// @return the length of the `buffers` vector + public int buffersLength() { + int o = fieldOffset(6); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `buffers` element + public FbsBuffer buffers(int j) { + int o = fieldOffset(6); + return new FbsBuffer().assign(seg, vectorElements(o) + (long) j * 8); + } + + /// Sets the `root` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addRoot(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Sets the `buffers` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addBuffers(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Begins the `buffers` vector. + /// @param b the builder + /// @param numElems element count + public static void startBuffersVector(FbsBuilder b, int numElems) { + b.startVector(8, numElems, 4); + } + + /// Builds a `FbsArray` table. + /// @param b the builder + /// @param rootOffset field value + /// @param buffersOffset field value + /// @return the table offset + public static int createFbsArray(FbsBuilder b, int rootOffset, int buffersOffset) { + b.startTable(2); + b.addOffset(1, buffersOffset, 0); + b.addOffset(0, rootOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsArray` table. + /// @param b the builder + public static void startFbsArray(FbsBuilder b) { + b.startTable(2); + } + + /// Finishes a `FbsArray` table. + /// @param b the builder + /// @return the table offset + public static int endFbsArray(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsArray` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsArrayBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayNode.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayNode.java new file mode 100644 index 000000000..d867d0954 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayNode.java @@ -0,0 +1,212 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `ArrayNode` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsArrayNode extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsArrayNode getRootAsFbsArrayNode(MemorySegment seg) { + return new FbsArrayNode().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsArrayNode assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `encoding` field + public int encoding() { + int o = fieldOffset(4); + return o != 0 ? readShort(pos + o) & 0xFFFF : 0; + } + + /// @return the length of the `metadata` vector + public int metadataLength() { + int o = fieldOffset(6); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `metadata` element + public int metadata(int j) { + int o = fieldOffset(6); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `metadata` byte vector, or null if absent + public MemorySegment metadataAsSegment() { + int o = fieldOffset(6); + return o != 0 ? byteVector(o) : null; + } + + /// @return the length of the `children` vector + public int childrenLength() { + int o = fieldOffset(8); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `children` element + public FbsArrayNode children(int j) { + int o = fieldOffset(8); + return new FbsArrayNode().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the length of the `buffers` vector + public int buffersLength() { + int o = fieldOffset(10); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `buffers` element + public int buffers(int j) { + int o = fieldOffset(10); + return o != 0 ? readShort(vectorElements(o) + (long) j * 2) & 0xFFFF : 0; + } + + /// @return the `stats` child, or null if absent + public FbsArrayStats stats() { + int o = fieldOffset(12); + return o != 0 ? new FbsArrayStats().assign(seg, indirect(pos + o)) : null; + } + + /// Sets the `encoding` field. + /// @param b the builder + /// @param encoding value + public static void addEncoding(FbsBuilder b, int encoding) { + b.addShort(0, (short) encoding, 0); + } + + /// Sets the `metadata` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addMetadata(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Creates the `metadata` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createMetadataVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `metadata` vector. + /// @param b the builder + /// @param numElems element count + public static void startMetadataVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Sets the `children` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addChildren(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Creates the `children` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createChildrenVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `children` vector. + /// @param b the builder + /// @param numElems element count + public static void startChildrenVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `buffers` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addBuffers(FbsBuilder b, int offset) { + b.addOffset(3, offset, 0); + } + + /// Creates the `buffers` vector. + /// @param b the builder + /// @param data the elements + /// @return the vector offset + public static int createBuffersVector(FbsBuilder b, int[] data) { + b.startVector(2, data.length, 2); + for (int i = data.length - 1; i >= 0; i--) { + b.addShort((short) data[i]); + } + return b.endVector(); + } + + /// Begins the `buffers` vector. + /// @param b the builder + /// @param numElems element count + public static void startBuffersVector(FbsBuilder b, int numElems) { + b.startVector(2, numElems, 2); + } + + /// Sets the `stats` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addStats(FbsBuilder b, int offset) { + b.addOffset(4, offset, 0); + } + + /// Builds a `FbsArrayNode` table. + /// @param b the builder + /// @param encoding field value + /// @param metadataOffset field value + /// @param childrenOffset field value + /// @param buffersOffset field value + /// @param statsOffset field value + /// @return the table offset + public static int createFbsArrayNode(FbsBuilder b, int encoding, int metadataOffset, int childrenOffset, int buffersOffset, int statsOffset) { + b.startTable(5); + b.addOffset(4, statsOffset, 0); + b.addOffset(3, buffersOffset, 0); + b.addOffset(2, childrenOffset, 0); + b.addOffset(1, metadataOffset, 0); + b.addShort(0, (short) encoding, 0); + return b.endTable(); + } + + /// Begins a `FbsArrayNode` table. + /// @param b the builder + public static void startFbsArrayNode(FbsBuilder b) { + b.startTable(5); + } + + /// Finishes a `FbsArrayNode` table. + /// @param b the builder + /// @return the table offset + public static int endFbsArrayNode(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsArrayNode` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsArrayNodeBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArraySpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArraySpec.java new file mode 100644 index 000000000..03b16e383 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArraySpec.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `ArraySpec` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsArraySpec extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsArraySpec getRootAsFbsArraySpec(MemorySegment seg) { + return new FbsArraySpec().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsArraySpec assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `id` field + public String id() { + int o = fieldOffset(4); + return o != 0 ? readStringAt(pos + o) : null; + } + + /// Sets the `id` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addId(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Builds a `FbsArraySpec` table. + /// @param b the builder + /// @param idOffset field value + /// @return the table offset + public static int createFbsArraySpec(FbsBuilder b, int idOffset) { + b.startTable(1); + b.addOffset(0, idOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsArraySpec` table. + /// @param b the builder + public static void startFbsArraySpec(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsArraySpec` table. + /// @param b the builder + /// @return the table offset + public static int endFbsArraySpec(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsArraySpec` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsArraySpecBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayStats.java new file mode 100644 index 000000000..85d8d1a3b --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsArrayStats.java @@ -0,0 +1,336 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `ArrayStats` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsArrayStats extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsArrayStats getRootAsFbsArrayStats(MemorySegment seg) { + return new FbsArrayStats().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsArrayStats assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the length of the `min` vector + public int minLength() { + int o = fieldOffset(4); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `min` element + public int min(int j) { + int o = fieldOffset(4); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `min` byte vector, or null if absent + public MemorySegment minAsSegment() { + int o = fieldOffset(4); + return o != 0 ? byteVector(o) : null; + } + + /// @return the `minPrecision` field + public int minPrecision() { + int o = fieldOffset(6); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// @return the length of the `max` vector + public int maxLength() { + int o = fieldOffset(8); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `max` element + public int max(int j) { + int o = fieldOffset(8); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `max` byte vector, or null if absent + public MemorySegment maxAsSegment() { + int o = fieldOffset(8); + return o != 0 ? byteVector(o) : null; + } + + /// @return the `maxPrecision` field + public int maxPrecision() { + int o = fieldOffset(10); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// @return the length of the `sum` vector + public int sumLength() { + int o = fieldOffset(12); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `sum` element + public int sum(int j) { + int o = fieldOffset(12); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `sum` byte vector, or null if absent + public MemorySegment sumAsSegment() { + int o = fieldOffset(12); + return o != 0 ? byteVector(o) : null; + } + + /// @return the `isSorted` field + public boolean isSorted() { + int o = fieldOffset(14); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// @return whether the optional `isSorted` field is present + public boolean hasIsSorted() { + return fieldOffset(14) != 0; + } + + /// @return the `isStrictSorted` field + public boolean isStrictSorted() { + int o = fieldOffset(16); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// @return whether the optional `isStrictSorted` field is present + public boolean hasIsStrictSorted() { + return fieldOffset(16) != 0; + } + + /// @return the `isConstant` field + public boolean isConstant() { + int o = fieldOffset(18); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// @return whether the optional `isConstant` field is present + public boolean hasIsConstant() { + return fieldOffset(18) != 0; + } + + /// @return the `nullCount` field + public long nullCount() { + int o = fieldOffset(20); + return o != 0 ? readLong(pos + o) : 0L; + } + + /// @return whether the optional `nullCount` field is present + public boolean hasNullCount() { + return fieldOffset(20) != 0; + } + + /// @return the `uncompressedSizeInBytes` field + public long uncompressedSizeInBytes() { + int o = fieldOffset(22); + return o != 0 ? readLong(pos + o) : 0L; + } + + /// @return whether the optional `uncompressedSizeInBytes` field is present + public boolean hasUncompressedSizeInBytes() { + return fieldOffset(22) != 0; + } + + /// @return the `nanCount` field + public long nanCount() { + int o = fieldOffset(24); + return o != 0 ? readLong(pos + o) : 0L; + } + + /// @return whether the optional `nanCount` field is present + public boolean hasNanCount() { + return fieldOffset(24) != 0; + } + + /// Sets the `min` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addMin(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Creates the `min` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createMinVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `min` vector. + /// @param b the builder + /// @param numElems element count + public static void startMinVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Sets the `minPrecision` field. + /// @param b the builder + /// @param minPrecision value + public static void addMinPrecision(FbsBuilder b, int minPrecision) { + b.addByte(1, (byte) minPrecision, 0); + } + + /// Sets the `max` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addMax(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Creates the `max` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createMaxVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `max` vector. + /// @param b the builder + /// @param numElems element count + public static void startMaxVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Sets the `maxPrecision` field. + /// @param b the builder + /// @param maxPrecision value + public static void addMaxPrecision(FbsBuilder b, int maxPrecision) { + b.addByte(3, (byte) maxPrecision, 0); + } + + /// Sets the `sum` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addSum(FbsBuilder b, int offset) { + b.addOffset(4, offset, 0); + } + + /// Creates the `sum` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createSumVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `sum` vector. + /// @param b the builder + /// @param numElems element count + public static void startSumVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Sets the `isSorted` field. + /// @param b the builder + /// @param isSorted value + public static void addIsSorted(FbsBuilder b, boolean isSorted) { + b.addBoolean(5, isSorted, false); + } + + /// Sets the `isStrictSorted` field. + /// @param b the builder + /// @param isStrictSorted value + public static void addIsStrictSorted(FbsBuilder b, boolean isStrictSorted) { + b.addBoolean(6, isStrictSorted, false); + } + + /// Sets the `isConstant` field. + /// @param b the builder + /// @param isConstant value + public static void addIsConstant(FbsBuilder b, boolean isConstant) { + b.addBoolean(7, isConstant, false); + } + + /// Sets the `nullCount` field. + /// @param b the builder + /// @param nullCount value + public static void addNullCount(FbsBuilder b, long nullCount) { + b.addLong(8, nullCount, 0L); + } + + /// Sets the `uncompressedSizeInBytes` field. + /// @param b the builder + /// @param uncompressedSizeInBytes value + public static void addUncompressedSizeInBytes(FbsBuilder b, long uncompressedSizeInBytes) { + b.addLong(9, uncompressedSizeInBytes, 0L); + } + + /// Sets the `nanCount` field. + /// @param b the builder + /// @param nanCount value + public static void addNanCount(FbsBuilder b, long nanCount) { + b.addLong(10, nanCount, 0L); + } + + /// Builds a `FbsArrayStats` table. + /// @param b the builder + /// @param minOffset field value + /// @param minPrecision field value + /// @param maxOffset field value + /// @param maxPrecision field value + /// @param sumOffset field value + /// @param isSorted field value + /// @param isStrictSorted field value + /// @param isConstant field value + /// @param nullCount field value + /// @param uncompressedSizeInBytes field value + /// @param nanCount field value + /// @return the table offset + public static int createFbsArrayStats(FbsBuilder b, int minOffset, int minPrecision, int maxOffset, int maxPrecision, int sumOffset, boolean isSorted, boolean isStrictSorted, boolean isConstant, long nullCount, long uncompressedSizeInBytes, long nanCount) { + b.startTable(11); + b.addLong(10, nanCount, 0L); + b.addLong(9, uncompressedSizeInBytes, 0L); + b.addLong(8, nullCount, 0L); + b.addOffset(4, sumOffset, 0); + b.addOffset(2, maxOffset, 0); + b.addOffset(0, minOffset, 0); + b.addBoolean(7, isConstant, false); + b.addBoolean(6, isStrictSorted, false); + b.addBoolean(5, isSorted, false); + b.addByte(3, (byte) maxPrecision, 0); + b.addByte(1, (byte) minPrecision, 0); + return b.endTable(); + } + + /// Begins a `FbsArrayStats` table. + /// @param b the builder + public static void startFbsArrayStats(FbsBuilder b) { + b.startTable(11); + } + + /// Finishes a `FbsArrayStats` table. + /// @param b the builder + /// @return the table offset + public static int endFbsArrayStats(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsArrayStats` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsArrayStatsBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBinary.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBinary.java new file mode 100644 index 000000000..cbb5d19b8 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBinary.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Binary` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsBinary extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsBinary getRootAsFbsBinary(MemorySegment seg) { + return new FbsBinary().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsBinary assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(0, nullable, false); + } + + /// Builds a `FbsBinary` table. + /// @param b the builder + /// @param nullable field value + /// @return the table offset + public static int createFbsBinary(FbsBuilder b, boolean nullable) { + b.startTable(1); + b.addBoolean(0, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsBinary` table. + /// @param b the builder + public static void startFbsBinary(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsBinary` table. + /// @param b the builder + /// @return the table offset + public static int endFbsBinary(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsBinary` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsBinaryBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBool.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBool.java new file mode 100644 index 000000000..450345487 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBool.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Bool` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsBool extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsBool getRootAsFbsBool(MemorySegment seg) { + return new FbsBool().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsBool assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(0, nullable, false); + } + + /// Builds a `FbsBool` table. + /// @param b the builder + /// @param nullable field value + /// @return the table offset + public static int createFbsBool(FbsBuilder b, boolean nullable) { + b.startTable(1); + b.addBoolean(0, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsBool` table. + /// @param b the builder + public static void startFbsBool(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsBool` table. + /// @param b the builder + /// @return the table offset + public static int endFbsBool(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsBool` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsBoolBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBuffer.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBuffer.java new file mode 100644 index 000000000..53c06d775 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsBuffer.java @@ -0,0 +1,58 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsStruct; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Buffer` FlatBuffers struct (inline, 8 bytes). +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsBuffer extends FbsStruct { + + /// Positions this reader at a struct. + /// @param seg the buffer + /// @param position the struct position + /// @return this + public FbsBuffer assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `padding` field + public int padding() { + return readShort(0) & 0xFFFF; + } + + /// @return the `alignmentExponent` field + public int alignmentExponent() { + return readByte(2) & 0xFF; + } + + /// @return the `compression` field + public int compression() { + return readByte(3) & 0xFF; + } + + /// @return the `length` field + public long length() { + return readInt(4) & 0xFFFFFFFFL; + } + + /// Writes an inline `FbsBuffer` struct. + /// @param b the builder + /// @param padding field value + /// @param alignmentExponent field value + /// @param compression field value + /// @param length field value + /// @return the struct offset + public static int createFbsBuffer(FbsBuilder b, int padding, int alignmentExponent, int compression, long length) { + b.prep(4, 8); + b.putInt((int) length); + b.putByte((byte) compression); + b.putByte((byte) alignmentExponent); + b.putShort((short) padding); + return b.offset(); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompression.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompression.java new file mode 100644 index 000000000..83aa998c8 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompression.java @@ -0,0 +1,19 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import javax.annotation.processing.Generated; + +/// Constants for the `Compression` FlatBuffers enum. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsCompression { + + private FbsCompression() { + } + + /// `None` = 0 + public static final byte None = (byte) 0; + + /// `LZ4` = 1 + public static final byte LZ4 = (byte) 1; +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionScheme.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionScheme.java new file mode 100644 index 000000000..fa00758e5 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionScheme.java @@ -0,0 +1,25 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import javax.annotation.processing.Generated; + +/// Constants for the `CompressionScheme` FlatBuffers enum. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsCompressionScheme { + + private FbsCompressionScheme() { + } + + /// `None` = 0 + public static final byte None = (byte) 0; + + /// `LZ4` = 1 + public static final byte LZ4 = (byte) 1; + + /// `ZLib` = 2 + public static final byte ZLib = (byte) 2; + + /// `ZStd` = 3 + public static final byte ZStd = (byte) 3; +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionSpec.java new file mode 100644 index 000000000..733886ffe --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsCompressionSpec.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `CompressionSpec` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsCompressionSpec extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsCompressionSpec getRootAsFbsCompressionSpec(MemorySegment seg) { + return new FbsCompressionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsCompressionSpec assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `scheme` field + public int scheme() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// Sets the `scheme` field. + /// @param b the builder + /// @param scheme value + public static void addScheme(FbsBuilder b, int scheme) { + b.addByte(0, (byte) scheme, 0); + } + + /// Builds a `FbsCompressionSpec` table. + /// @param b the builder + /// @param scheme field value + /// @return the table offset + public static int createFbsCompressionSpec(FbsBuilder b, int scheme) { + b.startTable(1); + b.addByte(0, (byte) scheme, 0); + return b.endTable(); + } + + /// Begins a `FbsCompressionSpec` table. + /// @param b the builder + public static void startFbsCompressionSpec(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsCompressionSpec` table. + /// @param b the builder + /// @return the table offset + public static int endFbsCompressionSpec(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsCompressionSpec` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsCompressionSpecBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDType.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDType.java new file mode 100644 index 000000000..1afeed142 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDType.java @@ -0,0 +1,90 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `DType` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsDType extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsDType getRootAsFbsDType(MemorySegment seg) { + return new FbsDType().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsDType assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the union discriminator for `type` (0 = none) + public int typeType() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// Projects the selected `type` member onto the given reader. + /// @param obj a reader for the expected member type + /// @param the member reader type + /// @return the positioned reader, or null if the union is absent + public T type(T obj) { + int o = fieldOffset(6); + return o != 0 ? locate(obj, unionMemberPosition(o)) : null; + } + + /// Sets the `typeType` field. + /// @param b the builder + /// @param value the value + public static void addTypeType(FbsBuilder b, byte value) { + b.addByte(0, value, 0); + } + + /// Sets the `type` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addType(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Builds a `FbsDType` table. + /// @param b the builder + /// @param typeType field value + /// @param typeOffset field value + /// @return the table offset + public static int createFbsDType(FbsBuilder b, byte typeType, int typeOffset) { + b.startTable(2); + b.addOffset(1, typeOffset, 0); + b.addByte(0, typeType, 0); + return b.endTable(); + } + + /// Begins a `FbsDType` table. + /// @param b the builder + public static void startFbsDType(FbsBuilder b) { + b.startTable(2); + } + + /// Finishes a `FbsDType` table. + /// @param b the builder + /// @return the table offset + public static int endFbsDType(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsDType` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsDTypeBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDecimal.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDecimal.java new file mode 100644 index 000000000..8e93aa6dc --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsDecimal.java @@ -0,0 +1,102 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Decimal` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsDecimal extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsDecimal getRootAsFbsDecimal(MemorySegment seg) { + return new FbsDecimal().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsDecimal assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `precision` field + public int precision() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// @return the `scale` field + public int scale() { + int o = fieldOffset(6); + return o != 0 ? (int) readByte(pos + o) : 0; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(8); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `precision` field. + /// @param b the builder + /// @param precision value + public static void addPrecision(FbsBuilder b, int precision) { + b.addByte(0, (byte) precision, 0); + } + + /// Sets the `scale` field. + /// @param b the builder + /// @param scale value + public static void addScale(FbsBuilder b, int scale) { + b.addByte(1, (byte) scale, 0); + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(2, nullable, false); + } + + /// Builds a `FbsDecimal` table. + /// @param b the builder + /// @param precision field value + /// @param scale field value + /// @param nullable field value + /// @return the table offset + public static int createFbsDecimal(FbsBuilder b, int precision, int scale, boolean nullable) { + b.startTable(3); + b.addBoolean(2, nullable, false); + b.addByte(1, (byte) scale, 0); + b.addByte(0, (byte) precision, 0); + return b.endTable(); + } + + /// Begins a `FbsDecimal` table. + /// @param b the builder + public static void startFbsDecimal(FbsBuilder b) { + b.startTable(3); + } + + /// Finishes a `FbsDecimal` table. + /// @param b the builder + /// @return the table offset + public static int endFbsDecimal(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsDecimal` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsDecimalBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsEncryptionSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsEncryptionSpec.java new file mode 100644 index 000000000..ccb2d4197 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsEncryptionSpec.java @@ -0,0 +1,57 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `EncryptionSpec` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsEncryptionSpec extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsEncryptionSpec getRootAsFbsEncryptionSpec(MemorySegment seg) { + return new FbsEncryptionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsEncryptionSpec assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// Builds a `FbsEncryptionSpec` table. + /// @param b the builder + /// @return the table offset + public static int createFbsEncryptionSpec(FbsBuilder b) { + b.startTable(0); + return b.endTable(); + } + + /// Begins a `FbsEncryptionSpec` table. + /// @param b the builder + public static void startFbsEncryptionSpec(FbsBuilder b) { + b.startTable(0); + } + + /// Finishes a `FbsEncryptionSpec` table. + /// @param b the builder + /// @return the table offset + public static int endFbsEncryptionSpec(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsEncryptionSpec` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsEncryptionSpecBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsExtension.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsExtension.java new file mode 100644 index 000000000..fec4f2b05 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsExtension.java @@ -0,0 +1,130 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Extension` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsExtension extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsExtension getRootAsFbsExtension(MemorySegment seg) { + return new FbsExtension().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsExtension assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `id` field + public String id() { + int o = fieldOffset(4); + return o != 0 ? readStringAt(pos + o) : null; + } + + /// @return the `storageDtype` child, or null if absent + public FbsDType storageDtype() { + int o = fieldOffset(6); + return o != 0 ? new FbsDType().assign(seg, indirect(pos + o)) : null; + } + + /// @return the length of the `metadata` vector + public int metadataLength() { + int o = fieldOffset(8); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `metadata` element + public int metadata(int j) { + int o = fieldOffset(8); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `metadata` byte vector, or null if absent + public MemorySegment metadataAsSegment() { + int o = fieldOffset(8); + return o != 0 ? byteVector(o) : null; + } + + /// Sets the `id` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addId(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Sets the `storageDtype` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addStorageDtype(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Sets the `metadata` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addMetadata(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Creates the `metadata` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createMetadataVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `metadata` vector. + /// @param b the builder + /// @param numElems element count + public static void startMetadataVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Builds a `FbsExtension` table. + /// @param b the builder + /// @param idOffset field value + /// @param storageDtypeOffset field value + /// @param metadataOffset field value + /// @return the table offset + public static int createFbsExtension(FbsBuilder b, int idOffset, int storageDtypeOffset, int metadataOffset) { + b.startTable(3); + b.addOffset(2, metadataOffset, 0); + b.addOffset(1, storageDtypeOffset, 0); + b.addOffset(0, idOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsExtension` table. + /// @param b the builder + public static void startFbsExtension(FbsBuilder b) { + b.startTable(3); + } + + /// Finishes a `FbsExtension` table. + /// @param b the builder + /// @return the table offset + public static int endFbsExtension(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsExtension` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsExtensionBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFileStatistics.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFileStatistics.java new file mode 100644 index 000000000..bc5c37e3a --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFileStatistics.java @@ -0,0 +1,98 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `FileStatistics` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsFileStatistics extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsFileStatistics getRootAsFbsFileStatistics(MemorySegment seg) { + return new FbsFileStatistics().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsFileStatistics assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the length of the `fieldStats` vector + public int fieldStatsLength() { + int o = fieldOffset(4); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `fieldStats` element + public FbsArrayStats fieldStats(int j) { + int o = fieldOffset(4); + return new FbsArrayStats().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// Sets the `fieldStats` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addFieldStats(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Creates the `fieldStats` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createFieldStatsVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `fieldStats` vector. + /// @param b the builder + /// @param numElems element count + public static void startFieldStatsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Builds a `FbsFileStatistics` table. + /// @param b the builder + /// @param fieldStatsOffset field value + /// @return the table offset + public static int createFbsFileStatistics(FbsBuilder b, int fieldStatsOffset) { + b.startTable(1); + b.addOffset(0, fieldStatsOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsFileStatistics` table. + /// @param b the builder + public static void startFbsFileStatistics(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsFileStatistics` table. + /// @param b the builder + /// @return the table offset + public static int endFbsFileStatistics(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsFileStatistics` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsFileStatisticsBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFixedSizeList.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFixedSizeList.java new file mode 100644 index 000000000..c46dca1b0 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFixedSizeList.java @@ -0,0 +1,102 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `FixedSizeList` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsFixedSizeList extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsFixedSizeList getRootAsFbsFixedSizeList(MemorySegment seg) { + return new FbsFixedSizeList().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsFixedSizeList assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `elementType` child, or null if absent + public FbsDType elementType() { + int o = fieldOffset(4); + return o != 0 ? new FbsDType().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `size` field + public long size() { + int o = fieldOffset(6); + return o != 0 ? readInt(pos + o) & 0xFFFFFFFFL : 0L; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(8); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `elementType` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addElementType(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Sets the `size` field. + /// @param b the builder + /// @param size value + public static void addSize(FbsBuilder b, long size) { + b.addInt(1, (int) size, 0); + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(2, nullable, false); + } + + /// Builds a `FbsFixedSizeList` table. + /// @param b the builder + /// @param elementTypeOffset field value + /// @param size field value + /// @param nullable field value + /// @return the table offset + public static int createFbsFixedSizeList(FbsBuilder b, int elementTypeOffset, long size, boolean nullable) { + b.startTable(3); + b.addInt(1, (int) size, 0); + b.addOffset(0, elementTypeOffset, 0); + b.addBoolean(2, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsFixedSizeList` table. + /// @param b the builder + public static void startFbsFixedSizeList(FbsBuilder b) { + b.startTable(3); + } + + /// Finishes a `FbsFixedSizeList` table. + /// @param b the builder + /// @return the table offset + public static int endFbsFixedSizeList(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsFixedSizeList` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsFixedSizeListBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFooter.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFooter.java new file mode 100644 index 000000000..9155a50c1 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsFooter.java @@ -0,0 +1,250 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Footer` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsFooter extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsFooter getRootAsFbsFooter(MemorySegment seg) { + return new FbsFooter().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsFooter assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the length of the `arraySpecs` vector + public int arraySpecsLength() { + int o = fieldOffset(4); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `arraySpecs` element + public FbsArraySpec arraySpecs(int j) { + int o = fieldOffset(4); + return new FbsArraySpec().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the length of the `layoutSpecs` vector + public int layoutSpecsLength() { + int o = fieldOffset(6); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `layoutSpecs` element + public FbsLayoutSpec layoutSpecs(int j) { + int o = fieldOffset(6); + return new FbsLayoutSpec().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the length of the `segmentSpecs` vector + public int segmentSpecsLength() { + int o = fieldOffset(8); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `segmentSpecs` element + public FbsSegmentSpec segmentSpecs(int j) { + int o = fieldOffset(8); + return new FbsSegmentSpec().assign(seg, vectorElements(o) + (long) j * 16); + } + + /// @return the length of the `compressionSpecs` vector + public int compressionSpecsLength() { + int o = fieldOffset(10); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `compressionSpecs` element + public FbsCompressionSpec compressionSpecs(int j) { + int o = fieldOffset(10); + return new FbsCompressionSpec().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the length of the `encryptionSpecs` vector + public int encryptionSpecsLength() { + int o = fieldOffset(12); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `encryptionSpecs` element + public FbsEncryptionSpec encryptionSpecs(int j) { + int o = fieldOffset(12); + return new FbsEncryptionSpec().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// Sets the `arraySpecs` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addArraySpecs(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Creates the `arraySpecs` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createArraySpecsVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `arraySpecs` vector. + /// @param b the builder + /// @param numElems element count + public static void startArraySpecsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `layoutSpecs` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addLayoutSpecs(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Creates the `layoutSpecs` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createLayoutSpecsVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `layoutSpecs` vector. + /// @param b the builder + /// @param numElems element count + public static void startLayoutSpecsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `segmentSpecs` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addSegmentSpecs(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Begins the `segmentSpecs` vector. + /// @param b the builder + /// @param numElems element count + public static void startSegmentSpecsVector(FbsBuilder b, int numElems) { + b.startVector(16, numElems, 8); + } + + /// Sets the `compressionSpecs` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addCompressionSpecs(FbsBuilder b, int offset) { + b.addOffset(3, offset, 0); + } + + /// Creates the `compressionSpecs` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createCompressionSpecsVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `compressionSpecs` vector. + /// @param b the builder + /// @param numElems element count + public static void startCompressionSpecsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `encryptionSpecs` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addEncryptionSpecs(FbsBuilder b, int offset) { + b.addOffset(4, offset, 0); + } + + /// Creates the `encryptionSpecs` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createEncryptionSpecsVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `encryptionSpecs` vector. + /// @param b the builder + /// @param numElems element count + public static void startEncryptionSpecsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Builds a `FbsFooter` table. + /// @param b the builder + /// @param arraySpecsOffset field value + /// @param layoutSpecsOffset field value + /// @param segmentSpecsOffset field value + /// @param compressionSpecsOffset field value + /// @param encryptionSpecsOffset field value + /// @return the table offset + public static int createFbsFooter(FbsBuilder b, int arraySpecsOffset, int layoutSpecsOffset, int segmentSpecsOffset, int compressionSpecsOffset, int encryptionSpecsOffset) { + b.startTable(5); + b.addOffset(4, encryptionSpecsOffset, 0); + b.addOffset(3, compressionSpecsOffset, 0); + b.addOffset(2, segmentSpecsOffset, 0); + b.addOffset(1, layoutSpecsOffset, 0); + b.addOffset(0, arraySpecsOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsFooter` table. + /// @param b the builder + public static void startFbsFooter(FbsBuilder b) { + b.startTable(5); + } + + /// Finishes a `FbsFooter` table. + /// @param b the builder + /// @return the table offset + public static int endFbsFooter(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsFooter` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsFooterBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayout.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayout.java new file mode 100644 index 000000000..666b46ca6 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayout.java @@ -0,0 +1,212 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Layout` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsLayout extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsLayout getRootAsFbsLayout(MemorySegment seg) { + return new FbsLayout().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsLayout assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `encoding` field + public int encoding() { + int o = fieldOffset(4); + return o != 0 ? readShort(pos + o) & 0xFFFF : 0; + } + + /// @return the `rowCount` field + public long rowCount() { + int o = fieldOffset(6); + return o != 0 ? readLong(pos + o) : 0L; + } + + /// @return the length of the `metadata` vector + public int metadataLength() { + int o = fieldOffset(8); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `metadata` element + public int metadata(int j) { + int o = fieldOffset(8); + return o != 0 ? readByte(vectorElements(o) + (long) j * 1) & 0xFF : 0; + } + + /// @return a zero-copy slice of the `metadata` byte vector, or null if absent + public MemorySegment metadataAsSegment() { + int o = fieldOffset(8); + return o != 0 ? byteVector(o) : null; + } + + /// @return the length of the `children` vector + public int childrenLength() { + int o = fieldOffset(10); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `children` element + public FbsLayout children(int j) { + int o = fieldOffset(10); + return new FbsLayout().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the length of the `segments` vector + public int segmentsLength() { + int o = fieldOffset(12); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `segments` element + public long segments(int j) { + int o = fieldOffset(12); + return o != 0 ? readInt(vectorElements(o) + (long) j * 4) & 0xFFFFFFFFL : 0L; + } + + /// Sets the `encoding` field. + /// @param b the builder + /// @param encoding value + public static void addEncoding(FbsBuilder b, int encoding) { + b.addShort(0, (short) encoding, 0); + } + + /// Sets the `rowCount` field. + /// @param b the builder + /// @param rowCount value + public static void addRowCount(FbsBuilder b, long rowCount) { + b.addLong(1, rowCount, 0L); + } + + /// Sets the `metadata` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addMetadata(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Creates the `metadata` byte vector. + /// @param b the builder + /// @param data the bytes + /// @return the vector offset + public static int createMetadataVector(FbsBuilder b, byte[] data) { + return b.createByteVector(data); + } + + /// Begins the `metadata` vector. + /// @param b the builder + /// @param numElems element count + public static void startMetadataVector(FbsBuilder b, int numElems) { + b.startVector(1, numElems, 1); + } + + /// Sets the `children` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addChildren(FbsBuilder b, int offset) { + b.addOffset(3, offset, 0); + } + + /// Creates the `children` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createChildrenVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `children` vector. + /// @param b the builder + /// @param numElems element count + public static void startChildrenVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `segments` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addSegments(FbsBuilder b, int offset) { + b.addOffset(4, offset, 0); + } + + /// Creates the `segments` vector. + /// @param b the builder + /// @param data the elements + /// @return the vector offset + public static int createSegmentsVector(FbsBuilder b, long[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addInt((int) data[i]); + } + return b.endVector(); + } + + /// Begins the `segments` vector. + /// @param b the builder + /// @param numElems element count + public static void startSegmentsVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Builds a `FbsLayout` table. + /// @param b the builder + /// @param encoding field value + /// @param rowCount field value + /// @param metadataOffset field value + /// @param childrenOffset field value + /// @param segmentsOffset field value + /// @return the table offset + public static int createFbsLayout(FbsBuilder b, int encoding, long rowCount, int metadataOffset, int childrenOffset, int segmentsOffset) { + b.startTable(5); + b.addLong(1, rowCount, 0L); + b.addOffset(4, segmentsOffset, 0); + b.addOffset(3, childrenOffset, 0); + b.addOffset(2, metadataOffset, 0); + b.addShort(0, (short) encoding, 0); + return b.endTable(); + } + + /// Begins a `FbsLayout` table. + /// @param b the builder + public static void startFbsLayout(FbsBuilder b) { + b.startTable(5); + } + + /// Finishes a `FbsLayout` table. + /// @param b the builder + /// @return the table offset + public static int endFbsLayout(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsLayout` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsLayoutBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayoutSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayoutSpec.java new file mode 100644 index 000000000..a7b7aae31 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsLayoutSpec.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `LayoutSpec` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsLayoutSpec extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsLayoutSpec getRootAsFbsLayoutSpec(MemorySegment seg) { + return new FbsLayoutSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsLayoutSpec assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `id` field + public String id() { + int o = fieldOffset(4); + return o != 0 ? readStringAt(pos + o) : null; + } + + /// Sets the `id` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addId(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Builds a `FbsLayoutSpec` table. + /// @param b the builder + /// @param idOffset field value + /// @return the table offset + public static int createFbsLayoutSpec(FbsBuilder b, int idOffset) { + b.startTable(1); + b.addOffset(0, idOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsLayoutSpec` table. + /// @param b the builder + public static void startFbsLayoutSpec(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsLayoutSpec` table. + /// @param b the builder + /// @return the table offset + public static int endFbsLayoutSpec(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsLayoutSpec` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsLayoutSpecBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsList.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsList.java new file mode 100644 index 000000000..9473ba2f2 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsList.java @@ -0,0 +1,87 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `List` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsList extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsList getRootAsFbsList(MemorySegment seg) { + return new FbsList().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsList assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `elementType` child, or null if absent + public FbsDType elementType() { + int o = fieldOffset(4); + return o != 0 ? new FbsDType().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(6); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `elementType` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addElementType(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(1, nullable, false); + } + + /// Builds a `FbsList` table. + /// @param b the builder + /// @param elementTypeOffset field value + /// @param nullable field value + /// @return the table offset + public static int createFbsList(FbsBuilder b, int elementTypeOffset, boolean nullable) { + b.startTable(2); + b.addOffset(0, elementTypeOffset, 0); + b.addBoolean(1, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsList` table. + /// @param b the builder + public static void startFbsList(FbsBuilder b) { + b.startTable(2); + } + + /// Finishes a `FbsList` table. + /// @param b the builder + /// @return the table offset + public static int endFbsList(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsList` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsListBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsNull.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsNull.java new file mode 100644 index 000000000..295a62a11 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsNull.java @@ -0,0 +1,57 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Null` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsNull extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsNull getRootAsFbsNull(MemorySegment seg) { + return new FbsNull().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsNull assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// Builds a `FbsNull` table. + /// @param b the builder + /// @return the table offset + public static int createFbsNull(FbsBuilder b) { + b.startTable(0); + return b.endTable(); + } + + /// Begins a `FbsNull` table. + /// @param b the builder + public static void startFbsNull(FbsBuilder b) { + b.startTable(0); + } + + /// Finishes a `FbsNull` table. + /// @param b the builder + /// @return the table offset + public static int endFbsNull(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsNull` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsNullBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPType.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPType.java new file mode 100644 index 000000000..0a9bdeb9f --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPType.java @@ -0,0 +1,46 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import javax.annotation.processing.Generated; + +/// Constants for the `PType` FlatBuffers enum. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsPType { + + private FbsPType() { + } + + /// `U8` = 0 + public static final byte U8 = (byte) 0; + + /// `U16` = 1 + public static final byte U16 = (byte) 1; + + /// `U32` = 2 + public static final byte U32 = (byte) 2; + + /// `U64` = 3 + public static final byte U64 = (byte) 3; + + /// `I8` = 4 + public static final byte I8 = (byte) 4; + + /// `I16` = 5 + public static final byte I16 = (byte) 5; + + /// `I32` = 6 + public static final byte I32 = (byte) 6; + + /// `I64` = 7 + public static final byte I64 = (byte) 7; + + /// `F16` = 8 + public static final byte F16 = (byte) 8; + + /// `F32` = 9 + public static final byte F32 = (byte) 9; + + /// `F64` = 10 + public static final byte F64 = (byte) 10; +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscript.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscript.java new file mode 100644 index 000000000..b0f670ef3 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscript.java @@ -0,0 +1,117 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Postscript` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsPostscript extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsPostscript getRootAsFbsPostscript(MemorySegment seg) { + return new FbsPostscript().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsPostscript assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `dtype` child, or null if absent + public FbsPostscriptSegment dtype() { + int o = fieldOffset(4); + return o != 0 ? new FbsPostscriptSegment().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `layout` child, or null if absent + public FbsPostscriptSegment layout() { + int o = fieldOffset(6); + return o != 0 ? new FbsPostscriptSegment().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `statistics` child, or null if absent + public FbsPostscriptSegment statistics() { + int o = fieldOffset(8); + return o != 0 ? new FbsPostscriptSegment().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `footer` child, or null if absent + public FbsPostscriptSegment footer() { + int o = fieldOffset(10); + return o != 0 ? new FbsPostscriptSegment().assign(seg, indirect(pos + o)) : null; + } + + /// Sets the `dtype` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addDtype(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Sets the `layout` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addLayout(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Sets the `statistics` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addStatistics(FbsBuilder b, int offset) { + b.addOffset(2, offset, 0); + } + + /// Sets the `footer` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addFooter(FbsBuilder b, int offset) { + b.addOffset(3, offset, 0); + } + + /// Builds a `FbsPostscript` table. + /// @param b the builder + /// @param dtypeOffset field value + /// @param layoutOffset field value + /// @param statisticsOffset field value + /// @param footerOffset field value + /// @return the table offset + public static int createFbsPostscript(FbsBuilder b, int dtypeOffset, int layoutOffset, int statisticsOffset, int footerOffset) { + b.startTable(4); + b.addOffset(3, footerOffset, 0); + b.addOffset(2, statisticsOffset, 0); + b.addOffset(1, layoutOffset, 0); + b.addOffset(0, dtypeOffset, 0); + return b.endTable(); + } + + /// Begins a `FbsPostscript` table. + /// @param b the builder + public static void startFbsPostscript(FbsBuilder b) { + b.startTable(4); + } + + /// Finishes a `FbsPostscript` table. + /// @param b the builder + /// @return the table offset + public static int endFbsPostscript(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsPostscript` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsPostscriptBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscriptSegment.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscriptSegment.java new file mode 100644 index 000000000..8d197d833 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPostscriptSegment.java @@ -0,0 +1,132 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `PostscriptSegment` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsPostscriptSegment extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsPostscriptSegment getRootAsFbsPostscriptSegment(MemorySegment seg) { + return new FbsPostscriptSegment().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsPostscriptSegment assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `offset` field + public long offset() { + int o = fieldOffset(4); + return o != 0 ? readLong(pos + o) : 0L; + } + + /// @return the `length` field + public long length() { + int o = fieldOffset(6); + return o != 0 ? readInt(pos + o) & 0xFFFFFFFFL : 0L; + } + + /// @return the `alignmentExponent` field + public int alignmentExponent() { + int o = fieldOffset(8); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// @return the `compression` child, or null if absent + public FbsCompressionSpec compression() { + int o = fieldOffset(10); + return o != 0 ? new FbsCompressionSpec().assign(seg, indirect(pos + o)) : null; + } + + /// @return the `encryption` child, or null if absent + public FbsEncryptionSpec encryption() { + int o = fieldOffset(12); + return o != 0 ? new FbsEncryptionSpec().assign(seg, indirect(pos + o)) : null; + } + + /// Sets the `offset` field. + /// @param b the builder + /// @param offset value + public static void addOffset(FbsBuilder b, long offset) { + b.addLong(0, offset, 0L); + } + + /// Sets the `length` field. + /// @param b the builder + /// @param length value + public static void addLength(FbsBuilder b, long length) { + b.addInt(1, (int) length, 0); + } + + /// Sets the `alignmentExponent` field. + /// @param b the builder + /// @param alignmentExponent value + public static void addAlignmentExponent(FbsBuilder b, int alignmentExponent) { + b.addByte(2, (byte) alignmentExponent, 0); + } + + /// Sets the `compression` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addCompression(FbsBuilder b, int offset) { + b.addOffset(3, offset, 0); + } + + /// Sets the `encryption` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addEncryption(FbsBuilder b, int offset) { + b.addOffset(4, offset, 0); + } + + /// Builds a `FbsPostscriptSegment` table. + /// @param b the builder + /// @param offset field value + /// @param length field value + /// @param alignmentExponent field value + /// @param compressionOffset field value + /// @param encryptionOffset field value + /// @return the table offset + public static int createFbsPostscriptSegment(FbsBuilder b, long offset, long length, int alignmentExponent, int compressionOffset, int encryptionOffset) { + b.startTable(5); + b.addLong(0, offset, 0L); + b.addOffset(4, encryptionOffset, 0); + b.addOffset(3, compressionOffset, 0); + b.addInt(1, (int) length, 0); + b.addByte(2, (byte) alignmentExponent, 0); + return b.endTable(); + } + + /// Begins a `FbsPostscriptSegment` table. + /// @param b the builder + public static void startFbsPostscriptSegment(FbsBuilder b) { + b.startTable(5); + } + + /// Finishes a `FbsPostscriptSegment` table. + /// @param b the builder + /// @return the table offset + public static int endFbsPostscriptSegment(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsPostscriptSegment` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsPostscriptSegmentBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrecision.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrecision.java new file mode 100644 index 000000000..22857c88f --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrecision.java @@ -0,0 +1,19 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import javax.annotation.processing.Generated; + +/// Constants for the `Precision` FlatBuffers enum. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsPrecision { + + private FbsPrecision() { + } + + /// `Inexact` = 0 + public static final byte Inexact = (byte) 0; + + /// `Exact` = 1 + public static final byte Exact = (byte) 1; +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrimitive.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrimitive.java new file mode 100644 index 000000000..ee51d67b6 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsPrimitive.java @@ -0,0 +1,87 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Primitive` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsPrimitive extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsPrimitive getRootAsFbsPrimitive(MemorySegment seg) { + return new FbsPrimitive().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsPrimitive assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `ptype` field + public int ptype() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) & 0xFF : 0; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(6); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `ptype` field. + /// @param b the builder + /// @param ptype value + public static void addPtype(FbsBuilder b, int ptype) { + b.addByte(0, (byte) ptype, 0); + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(1, nullable, false); + } + + /// Builds a `FbsPrimitive` table. + /// @param b the builder + /// @param ptype field value + /// @param nullable field value + /// @return the table offset + public static int createFbsPrimitive(FbsBuilder b, int ptype, boolean nullable) { + b.startTable(2); + b.addBoolean(1, nullable, false); + b.addByte(0, (byte) ptype, 0); + return b.endTable(); + } + + /// Begins a `FbsPrimitive` table. + /// @param b the builder + public static void startFbsPrimitive(FbsBuilder b) { + b.startTable(2); + } + + /// Finishes a `FbsPrimitive` table. + /// @param b the builder + /// @return the table offset + public static int endFbsPrimitive(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsPrimitive` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsPrimitiveBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsSegmentSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsSegmentSpec.java new file mode 100644 index 000000000..be3aab2f6 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsSegmentSpec.java @@ -0,0 +1,65 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsStruct; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `SegmentSpec` FlatBuffers struct (inline, 16 bytes). +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsSegmentSpec extends FbsStruct { + + /// Positions this reader at a struct. + /// @param seg the buffer + /// @param position the struct position + /// @return this + public FbsSegmentSpec assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `offset` field + public long offset() { + return readLong(0); + } + + /// @return the `length` field + public long length() { + return readInt(8) & 0xFFFFFFFFL; + } + + /// @return the `alignmentExponent` field + public int alignmentExponent() { + return readByte(12) & 0xFF; + } + + /// @return the `compression` field + public int compression() { + return readByte(13) & 0xFF; + } + + /// @return the `encryption` field + public int encryption() { + return readShort(14) & 0xFFFF; + } + + /// Writes an inline `FbsSegmentSpec` struct. + /// @param b the builder + /// @param offset field value + /// @param length field value + /// @param alignmentExponent field value + /// @param compression field value + /// @param encryption field value + /// @return the struct offset + public static int createFbsSegmentSpec(FbsBuilder b, long offset, long length, int alignmentExponent, int compression, int encryption) { + b.prep(8, 16); + b.putShort((short) encryption); + b.putByte((byte) compression); + b.putByte((byte) alignmentExponent); + b.putInt((int) length); + b.putLong(offset); + return b.offset(); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsStruct_.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsStruct_.java new file mode 100644 index 000000000..a88bbb884 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsStruct_.java @@ -0,0 +1,154 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Struct_` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsStruct_ extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsStruct_ getRootAsFbsStruct_(MemorySegment seg) { + return new FbsStruct_().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsStruct_ assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the length of the `names` vector + public int namesLength() { + int o = fieldOffset(4); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `names` string + public String names(int j) { + int o = fieldOffset(4); + return readStringAt(vectorElements(o) + (long) j * 4); + } + + /// @return the length of the `dtypes` vector + public int dtypesLength() { + int o = fieldOffset(6); + return o != 0 ? vectorLength(o) : 0; + } + + /// @param j element index + /// @return the j-th `dtypes` element + public FbsDType dtypes(int j) { + int o = fieldOffset(6); + return new FbsDType().assign(seg, indirect(vectorElements(o) + (long) j * 4)); + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(8); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `names` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addNames(FbsBuilder b, int offset) { + b.addOffset(0, offset, 0); + } + + /// Creates the `names` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createNamesVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `names` vector. + /// @param b the builder + /// @param numElems element count + public static void startNamesVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `dtypes` offset field. + /// @param b the builder + /// @param offset the referenced offset + public static void addDtypes(FbsBuilder b, int offset) { + b.addOffset(1, offset, 0); + } + + /// Creates the `dtypes` offset vector. + /// @param b the builder + /// @param data element offsets + /// @return the vector offset + public static int createDtypesVector(FbsBuilder b, int[] data) { + b.startVector(4, data.length, 4); + for (int i = data.length - 1; i >= 0; i--) { + b.addOffset(data[i]); + } + return b.endVector(); + } + + /// Begins the `dtypes` vector. + /// @param b the builder + /// @param numElems element count + public static void startDtypesVector(FbsBuilder b, int numElems) { + b.startVector(4, numElems, 4); + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(2, nullable, false); + } + + /// Builds a `FbsStruct_` table. + /// @param b the builder + /// @param namesOffset field value + /// @param dtypesOffset field value + /// @param nullable field value + /// @return the table offset + public static int createFbsStruct_(FbsBuilder b, int namesOffset, int dtypesOffset, boolean nullable) { + b.startTable(3); + b.addOffset(1, dtypesOffset, 0); + b.addOffset(0, namesOffset, 0); + b.addBoolean(2, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsStruct_` table. + /// @param b the builder + public static void startFbsStruct_(FbsBuilder b) { + b.startTable(3); + } + + /// Finishes a `FbsStruct_` table. + /// @param b the builder + /// @return the table offset + public static int endFbsStruct_(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsStruct_` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsStruct_Buffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsType.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsType.java new file mode 100644 index 000000000..aefb9ada3 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsType.java @@ -0,0 +1,52 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import javax.annotation.processing.Generated; + +/// Discriminator constants for the `Type` FlatBuffers union. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsType { + + private FbsType() { + } + + /// No member selected. + public static final byte NONE = 0; + + /// Member `Null` = 1 + public static final byte FbsNull = (byte) 1; + + /// Member `Bool` = 2 + public static final byte FbsBool = (byte) 2; + + /// Member `Primitive` = 3 + public static final byte FbsPrimitive = (byte) 3; + + /// Member `Decimal` = 4 + public static final byte FbsDecimal = (byte) 4; + + /// Member `Utf8` = 5 + public static final byte FbsUtf8 = (byte) 5; + + /// Member `Binary` = 6 + public static final byte FbsBinary = (byte) 6; + + /// Member `Struct_` = 7 + public static final byte FbsStruct_ = (byte) 7; + + /// Member `List` = 8 + public static final byte FbsList = (byte) 8; + + /// Member `Extension` = 9 + public static final byte FbsExtension = (byte) 9; + + /// Member `FixedSizeList` = 10 + public static final byte FbsFixedSizeList = (byte) 10; + + /// Member `Variant` = 11 + public static final byte FbsVariant = (byte) 11; + + /// Member `Union` = 12 + public static final byte FbsUnion = (byte) 12; +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUnion.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUnion.java new file mode 100644 index 000000000..8b1237690 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUnion.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Union` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsUnion extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsUnion getRootAsFbsUnion(MemorySegment seg) { + return new FbsUnion().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsUnion assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(0, nullable, false); + } + + /// Builds a `FbsUnion` table. + /// @param b the builder + /// @param nullable field value + /// @return the table offset + public static int createFbsUnion(FbsBuilder b, boolean nullable) { + b.startTable(1); + b.addBoolean(0, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsUnion` table. + /// @param b the builder + public static void startFbsUnion(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsUnion` table. + /// @param b the builder + /// @return the table offset + public static int endFbsUnion(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsUnion` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsUnionBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUtf8.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUtf8.java new file mode 100644 index 000000000..020ef8303 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsUtf8.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Utf8` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsUtf8 extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsUtf8 getRootAsFbsUtf8(MemorySegment seg) { + return new FbsUtf8().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsUtf8 assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(0, nullable, false); + } + + /// Builds a `FbsUtf8` table. + /// @param b the builder + /// @param nullable field value + /// @return the table offset + public static int createFbsUtf8(FbsBuilder b, boolean nullable) { + b.startTable(1); + b.addBoolean(0, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsUtf8` table. + /// @param b the builder + public static void startFbsUtf8(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsUtf8` table. + /// @param b the builder + /// @return the table offset + public static int endFbsUtf8(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsUtf8` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsUtf8Buffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FbsVariant.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsVariant.java new file mode 100644 index 000000000..8458fbce7 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbs/FbsVariant.java @@ -0,0 +1,72 @@ +// Generated by vortex-fbs-gen. Do not edit. + +package io.github.dfa1.vortex.fbs; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbsrt.FbsTable; +import java.lang.foreign.MemorySegment; +import javax.annotation.processing.Generated; + +/// Reader and builder for the `Variant` FlatBuffers table. +@Generated("io.github.dfa1.vortex.fbsgen.CodeGen") +public final class FbsVariant extends FbsTable { + + /// Positions a reader at the root table of a finished buffer. + /// @param seg the buffer + /// @return a reader positioned at the root + public static FbsVariant getRootAsFbsVariant(MemorySegment seg) { + return new FbsVariant().assign(seg, FbsTable.rootPosition(seg, 0)); + } + + /// Positions this reader at a table. + /// @param seg the buffer + /// @param position the table position + /// @return this + public FbsVariant assign(MemorySegment seg, long position) { + init(seg, position); + return this; + } + + /// @return the `nullable` field + public boolean nullable() { + int o = fieldOffset(4); + return o != 0 ? readByte(pos + o) != 0 : false; + } + + /// Sets the `nullable` field. + /// @param b the builder + /// @param nullable value + public static void addNullable(FbsBuilder b, boolean nullable) { + b.addBoolean(0, nullable, false); + } + + /// Builds a `FbsVariant` table. + /// @param b the builder + /// @param nullable field value + /// @return the table offset + public static int createFbsVariant(FbsBuilder b, boolean nullable) { + b.startTable(1); + b.addBoolean(0, nullable, false); + return b.endTable(); + } + + /// Begins a `FbsVariant` table. + /// @param b the builder + public static void startFbsVariant(FbsBuilder b) { + b.startTable(1); + } + + /// Finishes a `FbsVariant` table. + /// @param b the builder + /// @return the table offset + public static int endFbsVariant(FbsBuilder b) { + return b.endTable(); + } + + /// Finishes the buffer with a `FbsVariant` root. + /// @param b the builder + /// @param offset the root table offset + public static void finishFbsVariantBuffer(FbsBuilder b, int offset) { + b.finish(offset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FileStatistics.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FileStatistics.java deleted file mode 100644 index 29a237e84..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/FileStatistics.java +++ /dev/null @@ -1,106 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * The `FileStatistics` object contains file-level statistics for the Vortex file. - */ -@SuppressWarnings("unused") -public final class FileStatistics extends com.google.flatbuffers.Table { - public static FileStatistics getRootAsFileStatistics(ByteBuffer _bb) { - return getRootAsFileStatistics(_bb, new FileStatistics()); - } - - public static FileStatistics getRootAsFileStatistics(ByteBuffer _bb, FileStatistics obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createFileStatistics(FlatBufferBuilder builder, - int fieldStatsOffset) { - builder.startTable(1); - FileStatistics.addFieldStats(builder, fieldStatsOffset); - return FileStatistics.endFileStatistics(builder); - } - - public static void startFileStatistics(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addFieldStats(FlatBufferBuilder builder, int fieldStatsOffset) { - builder.addOffset(0, fieldStatsOffset, 0); - } - - public static int createFieldStatsVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startFieldStatsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static int endFileStatistics(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public FileStatistics __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * Statistics for each field in the root schema. If the root schema is not a struct, there will - * be a single entry in this array. - */ - public io.github.dfa1.vortex.fbs.ArrayStats fieldStats(int j) { - return fieldStats(new io.github.dfa1.vortex.fbs.ArrayStats(), j); - } - - public io.github.dfa1.vortex.fbs.ArrayStats fieldStats(io.github.dfa1.vortex.fbs.ArrayStats obj, int j) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int fieldStatsLength() { - int o = __offset(4); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.ArrayStats.Vector fieldStatsVector() { - return fieldStatsVector(new io.github.dfa1.vortex.fbs.ArrayStats.Vector()); - } - - public io.github.dfa1.vortex.fbs.ArrayStats.Vector fieldStatsVector(io.github.dfa1.vortex.fbs.ArrayStats.Vector obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public FileStatistics get(int j) { - return get(new FileStatistics(), j); - } - - public FileStatistics get(FileStatistics obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/FixedSizeList.java b/core/src/main/java/io/github/dfa1/vortex/fbs/FixedSizeList.java deleted file mode 100644 index 0f6313f7a..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/FixedSizeList.java +++ /dev/null @@ -1,97 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class FixedSizeList extends com.google.flatbuffers.Table { - public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb) { - return getRootAsFixedSizeList(_bb, new FixedSizeList()); - } - - public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb, FixedSizeList obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createFixedSizeList(FlatBufferBuilder builder, - int elementTypeOffset, - long size, - boolean nullable) { - builder.startTable(3); - FixedSizeList.addSize(builder, size); - FixedSizeList.addElementType(builder, elementTypeOffset); - FixedSizeList.addNullable(builder, nullable); - return FixedSizeList.endFixedSizeList(builder); - } - - public static void startFixedSizeList(FlatBufferBuilder builder) { - builder.startTable(3); - } - - public static void addElementType(FlatBufferBuilder builder, int elementTypeOffset) { - builder.addOffset(0, elementTypeOffset, 0); - } - - public static void addSize(FlatBufferBuilder builder, long size) { - builder.addInt(1, (int) size, (int) 0L); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(2, nullable, false); - } - - public static int endFixedSizeList(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public FixedSizeList __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public io.github.dfa1.vortex.fbs.DType elementType() { - return elementType(new io.github.dfa1.vortex.fbs.DType()); - } - - public io.github.dfa1.vortex.fbs.DType elementType(io.github.dfa1.vortex.fbs.DType obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public long size() { - int o = __offset(6); - return o != 0 ? (long) bb.getInt(o + bb_pos) & 0xFFFFFFFFL : 0L; - } - - public boolean nullable() { - int o = __offset(8); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public FixedSizeList get(int j) { - return get(new FixedSizeList(), j); - } - - public FixedSizeList get(FixedSizeList obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Footer.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Footer.java deleted file mode 100644 index 81b207f44..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Footer.java +++ /dev/null @@ -1,253 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * The `Registry` object stores dictionary-encoded configuration for segments, - * compression schemes, encryption schemes, etc. - */ -@SuppressWarnings("unused") -public final class Footer extends com.google.flatbuffers.Table { - public static Footer getRootAsFooter(ByteBuffer _bb) { - return getRootAsFooter(_bb, new Footer()); - } - - public static Footer getRootAsFooter(ByteBuffer _bb, Footer obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createFooter(FlatBufferBuilder builder, - int arraySpecsOffset, - int layoutSpecsOffset, - int segmentSpecsOffset, - int compressionSpecsOffset, - int encryptionSpecsOffset) { - builder.startTable(5); - Footer.addEncryptionSpecs(builder, encryptionSpecsOffset); - Footer.addCompressionSpecs(builder, compressionSpecsOffset); - Footer.addSegmentSpecs(builder, segmentSpecsOffset); - Footer.addLayoutSpecs(builder, layoutSpecsOffset); - Footer.addArraySpecs(builder, arraySpecsOffset); - return Footer.endFooter(builder); - } - - public static void startFooter(FlatBufferBuilder builder) { - builder.startTable(5); - } - - public static void addArraySpecs(FlatBufferBuilder builder, int arraySpecsOffset) { - builder.addOffset(0, arraySpecsOffset, 0); - } - - public static int createArraySpecsVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startArraySpecsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addLayoutSpecs(FlatBufferBuilder builder, int layoutSpecsOffset) { - builder.addOffset(1, layoutSpecsOffset, 0); - } - - public static int createLayoutSpecsVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startLayoutSpecsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addSegmentSpecs(FlatBufferBuilder builder, int segmentSpecsOffset) { - builder.addOffset(2, segmentSpecsOffset, 0); - } - - public static void startSegmentSpecsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(16, numElems, 8); - } - - public static void addCompressionSpecs(FlatBufferBuilder builder, int compressionSpecsOffset) { - builder.addOffset(3, compressionSpecsOffset, 0); - } - - public static int createCompressionSpecsVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startCompressionSpecsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addEncryptionSpecs(FlatBufferBuilder builder, int encryptionSpecsOffset) { - builder.addOffset(4, encryptionSpecsOffset, 0); - } - - public static int createEncryptionSpecsVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startEncryptionSpecsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static int endFooter(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Footer __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public io.github.dfa1.vortex.fbs.ArraySpec arraySpecs(int j) { - return arraySpecs(new io.github.dfa1.vortex.fbs.ArraySpec(), j); - } - - public io.github.dfa1.vortex.fbs.ArraySpec arraySpecs(io.github.dfa1.vortex.fbs.ArraySpec obj, int j) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int arraySpecsLength() { - int o = __offset(4); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.ArraySpec.Vector arraySpecsVector() { - return arraySpecsVector(new io.github.dfa1.vortex.fbs.ArraySpec.Vector()); - } - - public io.github.dfa1.vortex.fbs.ArraySpec.Vector arraySpecsVector(io.github.dfa1.vortex.fbs.ArraySpec.Vector obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public io.github.dfa1.vortex.fbs.LayoutSpec layoutSpecs(int j) { - return layoutSpecs(new io.github.dfa1.vortex.fbs.LayoutSpec(), j); - } - - public io.github.dfa1.vortex.fbs.LayoutSpec layoutSpecs(io.github.dfa1.vortex.fbs.LayoutSpec obj, int j) { - int o = __offset(6); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int layoutSpecsLength() { - int o = __offset(6); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.LayoutSpec.Vector layoutSpecsVector() { - return layoutSpecsVector(new io.github.dfa1.vortex.fbs.LayoutSpec.Vector()); - } - - public io.github.dfa1.vortex.fbs.LayoutSpec.Vector layoutSpecsVector(io.github.dfa1.vortex.fbs.LayoutSpec.Vector obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public io.github.dfa1.vortex.fbs.SegmentSpec segmentSpecs(int j) { - return segmentSpecs(new io.github.dfa1.vortex.fbs.SegmentSpec(), j); - } - - public io.github.dfa1.vortex.fbs.SegmentSpec segmentSpecs(io.github.dfa1.vortex.fbs.SegmentSpec obj, int j) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o) + j * 16, bb) : null; - } - - public int segmentSpecsLength() { - int o = __offset(8); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.SegmentSpec.Vector segmentSpecsVector() { - return segmentSpecsVector(new io.github.dfa1.vortex.fbs.SegmentSpec.Vector()); - } - - public io.github.dfa1.vortex.fbs.SegmentSpec.Vector segmentSpecsVector(io.github.dfa1.vortex.fbs.SegmentSpec.Vector obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o), 16, bb) : null; - } - - public io.github.dfa1.vortex.fbs.CompressionSpec compressionSpecs(int j) { - return compressionSpecs(new io.github.dfa1.vortex.fbs.CompressionSpec(), j); - } - - public io.github.dfa1.vortex.fbs.CompressionSpec compressionSpecs(io.github.dfa1.vortex.fbs.CompressionSpec obj, int j) { - int o = __offset(10); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int compressionSpecsLength() { - int o = __offset(10); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.CompressionSpec.Vector compressionSpecsVector() { - return compressionSpecsVector(new io.github.dfa1.vortex.fbs.CompressionSpec.Vector()); - } - - public io.github.dfa1.vortex.fbs.CompressionSpec.Vector compressionSpecsVector(io.github.dfa1.vortex.fbs.CompressionSpec.Vector obj) { - int o = __offset(10); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec encryptionSpecs(int j) { - return encryptionSpecs(new io.github.dfa1.vortex.fbs.EncryptionSpec(), j); - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec encryptionSpecs(io.github.dfa1.vortex.fbs.EncryptionSpec obj, int j) { - int o = __offset(12); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int encryptionSpecsLength() { - int o = __offset(12); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec.Vector encryptionSpecsVector() { - return encryptionSpecsVector(new io.github.dfa1.vortex.fbs.EncryptionSpec.Vector()); - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec.Vector encryptionSpecsVector(io.github.dfa1.vortex.fbs.EncryptionSpec.Vector obj) { - int o = __offset(12); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Footer get(int j) { - return get(new Footer(), j); - } - - public Footer get(Footer obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Layout.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Layout.java deleted file mode 100644 index 4f74e1c88..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Layout.java +++ /dev/null @@ -1,235 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.ByteVector; -import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.IntVector; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Layout extends com.google.flatbuffers.Table { - public static Layout getRootAsLayout(ByteBuffer _bb) { - return getRootAsLayout(_bb, new Layout()); - } - - public static Layout getRootAsLayout(ByteBuffer _bb, Layout obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createLayout(FlatBufferBuilder builder, - int encoding, - long rowCount, - int metadataOffset, - int childrenOffset, - int segmentsOffset) { - builder.startTable(5); - Layout.addRowCount(builder, rowCount); - Layout.addSegments(builder, segmentsOffset); - Layout.addChildren(builder, childrenOffset); - Layout.addMetadata(builder, metadataOffset); - Layout.addEncoding(builder, encoding); - return Layout.endLayout(builder); - } - - public static void startLayout(FlatBufferBuilder builder) { - builder.startTable(5); - } - - public static void addEncoding(FlatBufferBuilder builder, int encoding) { - builder.addShort(0, (short) encoding, (short) 0); - } - - public static void addRowCount(FlatBufferBuilder builder, long rowCount) { - builder.addLong(1, rowCount, 0L); - } - - public static void addMetadata(FlatBufferBuilder builder, int metadataOffset) { - builder.addOffset(2, metadataOffset, 0); - } - - public static int createMetadataVector(FlatBufferBuilder builder, byte[] data) { - return builder.createByteVector(data); - } - - public static int createMetadataVector(FlatBufferBuilder builder, ByteBuffer data) { - return builder.createByteVector(data); - } - - public static void startMetadataVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(1, numElems, 1); - } - - public static void addChildren(FlatBufferBuilder builder, int childrenOffset) { - builder.addOffset(3, childrenOffset, 0); - } - - public static int createChildrenVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startChildrenVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addSegments(FlatBufferBuilder builder, int segmentsOffset) { - builder.addOffset(4, segmentsOffset, 0); - } - - public static int createSegmentsVector(FlatBufferBuilder builder, long[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addInt((int) data[i]); - return builder.endVector(); - } - - public static void startSegmentsVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static int endLayout(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public static void finishLayoutBuffer(FlatBufferBuilder builder, int offset) { - builder.finish(offset); - } - - public static void finishSizePrefixedLayoutBuffer(FlatBufferBuilder builder, int offset) { - builder.finishSizePrefixed(offset); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Layout __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * The ID of the encoding used for this Layout. - */ - public int encoding() { - int o = __offset(4); - return o != 0 ? bb.getShort(o + bb_pos) & 0xFFFF : 0; - } - - /** - * The number of rows of data represented by this Layout. - */ - public long rowCount() { - int o = __offset(6); - return o != 0 ? bb.getLong(o + bb_pos) : 0L; - } - - /** - * Any additional metadata this layout needs to interpret its children. - * This does not include data-specific metadata, which the layout should store in a segment. - */ - public int metadata(int j) { - int o = __offset(8); - return o != 0 ? bb.get(__vector(o) + j * 1) & 0xFF : 0; - } - - public int metadataLength() { - int o = __offset(8); - return o != 0 ? __vector_len(o) : 0; - } - - public ByteVector metadataVector() { - return metadataVector(new ByteVector()); - } - - public ByteVector metadataVector(ByteVector obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer metadataAsByteBuffer() { - return __vector_as_bytebuffer(8, 1); - } - - public ByteBuffer metadataInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 8, 1); - } - - /** - * The children of this Layout. - */ - public io.github.dfa1.vortex.fbs.Layout children(int j) { - return children(new io.github.dfa1.vortex.fbs.Layout(), j); - } - - public io.github.dfa1.vortex.fbs.Layout children(io.github.dfa1.vortex.fbs.Layout obj, int j) { - int o = __offset(10); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int childrenLength() { - int o = __offset(10); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.Layout.Vector childrenVector() { - return childrenVector(new io.github.dfa1.vortex.fbs.Layout.Vector()); - } - - public io.github.dfa1.vortex.fbs.Layout.Vector childrenVector(io.github.dfa1.vortex.fbs.Layout.Vector obj) { - int o = __offset(10); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - /** - * Identifiers for each `SegmentSpec` of data required by this layout. - */ - public long segments(int j) { - int o = __offset(12); - return o != 0 ? (long) bb.getInt(__vector(o) + j * 4) & 0xFFFFFFFFL : 0; - } - - public int segmentsLength() { - int o = __offset(12); - return o != 0 ? __vector_len(o) : 0; - } - - public IntVector segmentsVector() { - return segmentsVector(new IntVector()); - } - - public IntVector segmentsVector(IntVector obj) { - int o = __offset(12); - return o != 0 ? obj.__assign(__vector(o), bb) : null; - } - - public ByteBuffer segmentsAsByteBuffer() { - return __vector_as_bytebuffer(12, 4); - } - - public ByteBuffer segmentsInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 12, 4); - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Layout get(int j) { - return get(new Layout(), j); - } - - public Layout get(Layout obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/LayoutSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/LayoutSpec.java deleted file mode 100644 index 40e2dd6e1..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/LayoutSpec.java +++ /dev/null @@ -1,86 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * A `LayoutSpec` describes the type of a particular layout. - *

- * These are identified by a globally unique string identifier, and looked up in the Vortex registry - * at read-time. - */ -@SuppressWarnings("unused") -public final class LayoutSpec extends com.google.flatbuffers.Table { - public static LayoutSpec getRootAsLayoutSpec(ByteBuffer _bb) { - return getRootAsLayoutSpec(_bb, new LayoutSpec()); - } - - public static LayoutSpec getRootAsLayoutSpec(ByteBuffer _bb, LayoutSpec obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createLayoutSpec(FlatBufferBuilder builder, - int idOffset) { - builder.startTable(1); - LayoutSpec.addId(builder, idOffset); - return LayoutSpec.endLayoutSpec(builder); - } - - public static void startLayoutSpec(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addId(FlatBufferBuilder builder, int idOffset) { - builder.addOffset(0, idOffset, 0); - } - - public static int endLayoutSpec(FlatBufferBuilder builder) { - int o = builder.endTable(); - builder.required(o, 4); // id - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public LayoutSpec __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public String id() { - int o = __offset(4); - return o != 0 ? __string(o + bb_pos) : null; - } - - public ByteBuffer idAsByteBuffer() { - return __vector_as_bytebuffer(4, 1); - } - - public ByteBuffer idInByteBuffer(ByteBuffer _bb) { - return __vector_in_bytebuffer(_bb, 4, 1); - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public LayoutSpec get(int j) { - return get(new LayoutSpec(), j); - } - - public LayoutSpec get(LayoutSpec obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/List.java b/core/src/main/java/io/github/dfa1/vortex/fbs/List.java deleted file mode 100644 index c8ee6c7f1..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/List.java +++ /dev/null @@ -1,86 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class List extends com.google.flatbuffers.Table { - public static List getRootAsList(ByteBuffer _bb) { - return getRootAsList(_bb, new List()); - } - - public static List getRootAsList(ByteBuffer _bb, List obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createList(FlatBufferBuilder builder, - int elementTypeOffset, - boolean nullable) { - builder.startTable(2); - List.addElementType(builder, elementTypeOffset); - List.addNullable(builder, nullable); - return List.endList(builder); - } - - public static void startList(FlatBufferBuilder builder) { - builder.startTable(2); - } - - public static void addElementType(FlatBufferBuilder builder, int elementTypeOffset) { - builder.addOffset(0, elementTypeOffset, 0); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(1, nullable, false); - } - - public static int endList(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public List __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public io.github.dfa1.vortex.fbs.DType elementType() { - return elementType(new io.github.dfa1.vortex.fbs.DType()); - } - - public io.github.dfa1.vortex.fbs.DType elementType(io.github.dfa1.vortex.fbs.DType obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public boolean nullable() { - int o = __offset(6); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public List get(int j) { - return get(new List(), j); - } - - public List get(List obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Null.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Null.java deleted file mode 100644 index 0bd55f55d..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Null.java +++ /dev/null @@ -1,55 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Null extends com.google.flatbuffers.Table { - public static Null getRootAsNull(ByteBuffer _bb) { - return getRootAsNull(_bb, new Null()); - } - - public static Null getRootAsNull(ByteBuffer _bb, Null obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static void startNull(FlatBufferBuilder builder) { - builder.startTable(0); - } - - public static int endNull(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Null __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Null get(int j) { - return get(new Null(), j); - } - - public Null get(Null obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/PType.java b/core/src/main/java/io/github/dfa1/vortex/fbs/PType.java deleted file mode 100644 index 6f79249e9..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/PType.java +++ /dev/null @@ -1,27 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -@SuppressWarnings("unused") -public final class PType { - public static final int U8 = 0; - public static final int U16 = 1; - public static final int U32 = 2; - public static final int U64 = 3; - public static final int I8 = 4; - public static final int I16 = 5; - public static final int I32 = 6; - public static final int I64 = 7; - public static final int F16 = 8; - public static final int F32 = 9; - public static final int F64 = 10; - public static final String[] names = {"U8", "U16", "U32", "U64", "I8", "I16", "I32", "I64", "F16", "F32", "F64",}; - - private PType() { - } - - public static String name(int e) { - return names[e]; - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Postscript.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Postscript.java deleted file mode 100644 index b64e7c6bf..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Postscript.java +++ /dev/null @@ -1,157 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * The `Postscript` is guaranteed by the file format to never exceed - * 65528 bytes (i.e., u16::MAX - 8 bytes) in length, and is immediately - * followed by an 8-byte `EndOfFile` struct. - *

- * An initial read of a Vortex file defaults to at least 64KB (u16::MAX bytes) and therefore - * is guaranteed to cover at least the Postscript. - *

- * The reason for a postscript at all is to ensure minimal but all necessary footer information - * can be read in two round trips. Since the DType is optional and possibly large, it lives in - * its own segment. If the footer were arbitrary size, with a pointer to the DType segment, then - * in the worst case we would need one round trip to read the footer length, one to read the full - * footer and parse the DType offset, and a third to fetch the DType segment. - *

- * The segments pointed to by the postscript have inline compression and encryption specs to avoid - * the need to fetch encryption schemes up-front. - */ -@SuppressWarnings("unused") -public final class Postscript extends com.google.flatbuffers.Table { - public static Postscript getRootAsPostscript(ByteBuffer _bb) { - return getRootAsPostscript(_bb, new Postscript()); - } - - public static Postscript getRootAsPostscript(ByteBuffer _bb, Postscript obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createPostscript(FlatBufferBuilder builder, - int dtypeOffset, - int layoutOffset, - int statisticsOffset, - int footerOffset) { - builder.startTable(4); - Postscript.addFooter(builder, footerOffset); - Postscript.addStatistics(builder, statisticsOffset); - Postscript.addLayout(builder, layoutOffset); - Postscript.addDtype(builder, dtypeOffset); - return Postscript.endPostscript(builder); - } - - public static void startPostscript(FlatBufferBuilder builder) { - builder.startTable(4); - } - - public static void addDtype(FlatBufferBuilder builder, int dtypeOffset) { - builder.addOffset(0, dtypeOffset, 0); - } - - public static void addLayout(FlatBufferBuilder builder, int layoutOffset) { - builder.addOffset(1, layoutOffset, 0); - } - - public static void addStatistics(FlatBufferBuilder builder, int statisticsOffset) { - builder.addOffset(2, statisticsOffset, 0); - } - - public static void addFooter(FlatBufferBuilder builder, int footerOffset) { - builder.addOffset(3, footerOffset, 0); - } - - public static int endPostscript(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public static void finishPostscriptBuffer(FlatBufferBuilder builder, int offset) { - builder.finish(offset); - } - - public static void finishSizePrefixedPostscriptBuffer(FlatBufferBuilder builder, int offset) { - builder.finishSizePrefixed(offset); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Postscript __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * Segment containing the root `DType` flatbuffer. - */ - public io.github.dfa1.vortex.fbs.PostscriptSegment dtype() { - return dtype(new io.github.dfa1.vortex.fbs.PostscriptSegment()); - } - - public io.github.dfa1.vortex.fbs.PostscriptSegment dtype(io.github.dfa1.vortex.fbs.PostscriptSegment obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - /** - * Segment containing the root `Layout` flatbuffer (required). - */ - public io.github.dfa1.vortex.fbs.PostscriptSegment layout() { - return layout(new io.github.dfa1.vortex.fbs.PostscriptSegment()); - } - - public io.github.dfa1.vortex.fbs.PostscriptSegment layout(io.github.dfa1.vortex.fbs.PostscriptSegment obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - /** - * Segment containing the file-level `Statistics` flatbuffer. - */ - public io.github.dfa1.vortex.fbs.PostscriptSegment statistics() { - return statistics(new io.github.dfa1.vortex.fbs.PostscriptSegment()); - } - - public io.github.dfa1.vortex.fbs.PostscriptSegment statistics(io.github.dfa1.vortex.fbs.PostscriptSegment obj) { - int o = __offset(8); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - /** - * Segment containing the 'Footer' flatbuffer (required) - */ - public io.github.dfa1.vortex.fbs.PostscriptSegment footer() { - return footer(new io.github.dfa1.vortex.fbs.PostscriptSegment()); - } - - public io.github.dfa1.vortex.fbs.PostscriptSegment footer(io.github.dfa1.vortex.fbs.PostscriptSegment obj) { - int o = __offset(10); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Postscript get(int j) { - return get(new Postscript(), j); - } - - public Postscript get(Postscript obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/PostscriptSegment.java b/core/src/main/java/io/github/dfa1/vortex/fbs/PostscriptSegment.java deleted file mode 100644 index b5f7b8907..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/PostscriptSegment.java +++ /dev/null @@ -1,127 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * A `PostscriptSegment` describes the location of a segment in the file without referencing any - * specification objects. That is, encryption and compression are defined inline. - */ -@SuppressWarnings("unused") -public final class PostscriptSegment extends com.google.flatbuffers.Table { - public static PostscriptSegment getRootAsPostscriptSegment(ByteBuffer _bb) { - return getRootAsPostscriptSegment(_bb, new PostscriptSegment()); - } - - public static PostscriptSegment getRootAsPostscriptSegment(ByteBuffer _bb, PostscriptSegment obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createPostscriptSegment(FlatBufferBuilder builder, - long offset, - long length, - int alignmentExponent, - int _CompressionOffset, - int _EncryptionOffset) { - builder.startTable(5); - PostscriptSegment.addOffset(builder, offset); - PostscriptSegment.add_encryption(builder, _EncryptionOffset); - PostscriptSegment.add_compression(builder, _CompressionOffset); - PostscriptSegment.addLength(builder, length); - PostscriptSegment.addAlignmentExponent(builder, alignmentExponent); - return PostscriptSegment.endPostscriptSegment(builder); - } - - public static void startPostscriptSegment(FlatBufferBuilder builder) { - builder.startTable(5); - } - - public static void addOffset(FlatBufferBuilder builder, long offset) { - builder.addLong(0, offset, 0L); - } - - public static void addLength(FlatBufferBuilder builder, long length) { - builder.addInt(1, (int) length, (int) 0L); - } - - public static void addAlignmentExponent(FlatBufferBuilder builder, int alignmentExponent) { - builder.addByte(2, (byte) alignmentExponent, (byte) 0); - } - - public static void add_compression(FlatBufferBuilder builder, int _CompressionOffset) { - builder.addOffset(3, _CompressionOffset, 0); - } - - public static void add_encryption(FlatBufferBuilder builder, int _EncryptionOffset) { - builder.addOffset(4, _EncryptionOffset, 0); - } - - public static int endPostscriptSegment(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public PostscriptSegment __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public long offset() { - int o = __offset(4); - return o != 0 ? bb.getLong(o + bb_pos) : 0L; - } - - public long length() { - int o = __offset(6); - return o != 0 ? (long) bb.getInt(o + bb_pos) & 0xFFFFFFFFL : 0L; - } - - public int alignmentExponent() { - int o = __offset(8); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public io.github.dfa1.vortex.fbs.CompressionSpec _Compression() { - return _Compression(new io.github.dfa1.vortex.fbs.CompressionSpec()); - } - - public io.github.dfa1.vortex.fbs.CompressionSpec _Compression(io.github.dfa1.vortex.fbs.CompressionSpec obj) { - int o = __offset(10); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec _Encryption() { - return _Encryption(new io.github.dfa1.vortex.fbs.EncryptionSpec()); - } - - public io.github.dfa1.vortex.fbs.EncryptionSpec _Encryption(io.github.dfa1.vortex.fbs.EncryptionSpec obj) { - int o = __offset(12); - return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public PostscriptSegment get(int j) { - return get(new PostscriptSegment(), j); - } - - public PostscriptSegment get(PostscriptSegment obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Precision.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Precision.java deleted file mode 100644 index fb54bc471..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Precision.java +++ /dev/null @@ -1,18 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -@SuppressWarnings("unused") -public final class Precision { - public static final int Inexact = 0; - public static final int Exact = 1; - public static final String[] names = {"Inexact", "Exact",}; - - private Precision() { - } - - public static String name(int e) { - return names[e]; - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Primitive.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Primitive.java deleted file mode 100644 index ce0032423..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Primitive.java +++ /dev/null @@ -1,82 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Primitive extends com.google.flatbuffers.Table { - public static Primitive getRootAsPrimitive(ByteBuffer _bb) { - return getRootAsPrimitive(_bb, new Primitive()); - } - - public static Primitive getRootAsPrimitive(ByteBuffer _bb, Primitive obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createPrimitive(FlatBufferBuilder builder, - int ptype, - boolean nullable) { - builder.startTable(2); - Primitive.addNullable(builder, nullable); - Primitive.addPtype(builder, ptype); - return Primitive.endPrimitive(builder); - } - - public static void startPrimitive(FlatBufferBuilder builder) { - builder.startTable(2); - } - - public static void addPtype(FlatBufferBuilder builder, int ptype) { - builder.addByte(0, (byte) ptype, (byte) 0); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(1, nullable, false); - } - - public static int endPrimitive(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Primitive __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public int ptype() { - int o = __offset(4); - return o != 0 ? bb.get(o + bb_pos) & 0xFF : 0; - } - - public boolean nullable() { - int o = __offset(6); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Primitive get(int j) { - return get(new Primitive(), j); - } - - public Primitive get(Primitive obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/SegmentSpec.java b/core/src/main/java/io/github/dfa1/vortex/fbs/SegmentSpec.java deleted file mode 100644 index 99ea30d7f..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/SegmentSpec.java +++ /dev/null @@ -1,79 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.Struct; - -import java.nio.ByteBuffer; - -/** - * A `SegmentSpec` acts as the locator for a buffer within the file. - */ -@SuppressWarnings("unused") -public final class SegmentSpec extends Struct { - public static int createSegmentSpec(FlatBufferBuilder builder, long offset, long length, int alignmentExponent, int _Compression, int _Encryption) { - builder.prep(8, 16); - builder.putShort((short) _Encryption); - builder.putByte((byte) _Compression); - builder.putByte((byte) alignmentExponent); - builder.putInt((int) length); - builder.putLong(offset); - return builder.offset(); - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public SegmentSpec __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - /** - * Offset relative to the start of the file. - */ - public long offset() { - return bb.getLong(bb_pos + 0); - } - - /** - * Length in bytes of the segment. - */ - public long length() { - return (long) bb.getInt(bb_pos + 8) & 0xFFFFFFFFL; - } - - /** - * Base-2 exponent of the alignment of the segment. - */ - public int alignmentExponent() { - return bb.get(bb_pos + 12) & 0xFF; - } - - public int _Compression() { - return bb.get(bb_pos + 13) & 0xFF; - } - - public int _Encryption() { - return bb.getShort(bb_pos + 14) & 0xFFFF; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public SegmentSpec get(int j) { - return get(new SegmentSpec(), j); - } - - public SegmentSpec get(SegmentSpec obj, int j) { - return obj.__assign(__element(j), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Struct_.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Struct_.java deleted file mode 100644 index bbae9bbb4..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Struct_.java +++ /dev/null @@ -1,146 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; -import com.google.flatbuffers.StringVector; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Struct_ extends com.google.flatbuffers.Table { - public static Struct_ getRootAsStruct_(ByteBuffer _bb) { - return getRootAsStruct_(_bb, new Struct_()); - } - - public static Struct_ getRootAsStruct_(ByteBuffer _bb, Struct_ obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createStruct_(FlatBufferBuilder builder, - int namesOffset, - int dtypesOffset, - boolean nullable) { - builder.startTable(3); - Struct_.addDtypes(builder, dtypesOffset); - Struct_.addNames(builder, namesOffset); - Struct_.addNullable(builder, nullable); - return Struct_.endStruct_(builder); - } - - public static void startStruct_(FlatBufferBuilder builder) { - builder.startTable(3); - } - - public static void addNames(FlatBufferBuilder builder, int namesOffset) { - builder.addOffset(0, namesOffset, 0); - } - - public static int createNamesVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startNamesVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addDtypes(FlatBufferBuilder builder, int dtypesOffset) { - builder.addOffset(1, dtypesOffset, 0); - } - - public static int createDtypesVector(FlatBufferBuilder builder, int[] data) { - builder.startVector(4, data.length, 4); - for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); - return builder.endVector(); - } - - public static void startDtypesVector(FlatBufferBuilder builder, int numElems) { - builder.startVector(4, numElems, 4); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(2, nullable, false); - } - - public static int endStruct_(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Struct_ __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public String names(int j) { - int o = __offset(4); - return o != 0 ? __string(__vector(o) + j * 4) : null; - } - - public int namesLength() { - int o = __offset(4); - return o != 0 ? __vector_len(o) : 0; - } - - public StringVector namesVector() { - return namesVector(new StringVector()); - } - - public StringVector namesVector(StringVector obj) { - int o = __offset(4); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public io.github.dfa1.vortex.fbs.DType dtypes(int j) { - return dtypes(new io.github.dfa1.vortex.fbs.DType(), j); - } - - public io.github.dfa1.vortex.fbs.DType dtypes(io.github.dfa1.vortex.fbs.DType obj, int j) { - int o = __offset(6); - return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; - } - - public int dtypesLength() { - int o = __offset(6); - return o != 0 ? __vector_len(o) : 0; - } - - public io.github.dfa1.vortex.fbs.DType.Vector dtypesVector() { - return dtypesVector(new io.github.dfa1.vortex.fbs.DType.Vector()); - } - - public io.github.dfa1.vortex.fbs.DType.Vector dtypesVector(io.github.dfa1.vortex.fbs.DType.Vector obj) { - int o = __offset(6); - return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; - } - - public boolean nullable() { - int o = __offset(8); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Struct_ get(int j) { - return get(new Struct_(), j); - } - - public Struct_ get(Struct_ obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Type.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Type.java deleted file mode 100644 index 099d5051c..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Type.java +++ /dev/null @@ -1,29 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -@SuppressWarnings("unused") -public final class Type { - public static final byte NONE = 0; - public static final byte Null = 1; - public static final byte Bool = 2; - public static final byte Primitive = 3; - public static final byte Decimal = 4; - public static final byte Utf8 = 5; - public static final byte Binary = 6; - public static final byte Struct_ = 7; - public static final byte List = 8; - public static final byte Extension = 9; - public static final byte FixedSizeList = 10; - public static final byte Variant = 11; - public static final byte Union = 12; - public static final String[] names = {"NONE", "Null", "Bool", "Primitive", "Decimal", "Utf8", "Binary", "Struct_", "List", "Extension", "FixedSizeList", "Variant", "Union",}; - - private Type() { - } - - public static String name(int e) { - return names[e]; - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Union.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Union.java deleted file mode 100644 index b6412725c..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Union.java +++ /dev/null @@ -1,71 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Union extends com.google.flatbuffers.Table { - public static Union getRootAsUnion(ByteBuffer _bb) { - return getRootAsUnion(_bb, new Union()); - } - - public static Union getRootAsUnion(ByteBuffer _bb, Union obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createUnion(FlatBufferBuilder builder, - boolean nullable) { - builder.startTable(1); - Union.addNullable(builder, nullable); - return Union.endUnion(builder); - } - - public static void startUnion(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(0, nullable, false); - } - - public static int endUnion(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Union __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public boolean nullable() { - int o = __offset(4); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Union get(int j) { - return get(new Union(), j); - } - - public Union get(Union obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Utf8.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Utf8.java deleted file mode 100644 index 8de99e2c0..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Utf8.java +++ /dev/null @@ -1,71 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Utf8 extends com.google.flatbuffers.Table { - public static Utf8 getRootAsUtf8(ByteBuffer _bb) { - return getRootAsUtf8(_bb, new Utf8()); - } - - public static Utf8 getRootAsUtf8(ByteBuffer _bb, Utf8 obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createUtf8(FlatBufferBuilder builder, - boolean nullable) { - builder.startTable(1); - Utf8.addNullable(builder, nullable); - return Utf8.endUtf8(builder); - } - - public static void startUtf8(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(0, nullable, false); - } - - public static int endUtf8(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Utf8 __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public boolean nullable() { - int o = __offset(4); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Utf8 get(int j) { - return get(new Utf8(), j); - } - - public Utf8 get(Utf8 obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbs/Variant.java b/core/src/main/java/io/github/dfa1/vortex/fbs/Variant.java deleted file mode 100644 index dbf6a8588..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/fbs/Variant.java +++ /dev/null @@ -1,71 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -package io.github.dfa1.vortex.fbs; - -import com.google.flatbuffers.BaseVector; -import com.google.flatbuffers.FlatBufferBuilder; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@SuppressWarnings("unused") -public final class Variant extends com.google.flatbuffers.Table { - public static Variant getRootAsVariant(ByteBuffer _bb) { - return getRootAsVariant(_bb, new Variant()); - } - - public static Variant getRootAsVariant(ByteBuffer _bb, Variant obj) { - _bb.order(ByteOrder.LITTLE_ENDIAN); - return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); - } - - public static int createVariant(FlatBufferBuilder builder, - boolean nullable) { - builder.startTable(1); - Variant.addNullable(builder, nullable); - return Variant.endVariant(builder); - } - - public static void startVariant(FlatBufferBuilder builder) { - builder.startTable(1); - } - - public static void addNullable(FlatBufferBuilder builder, boolean nullable) { - builder.addBoolean(0, nullable, false); - } - - public static int endVariant(FlatBufferBuilder builder) { - int o = builder.endTable(); - return o; - } - - public void __init(int _i, ByteBuffer _bb) { - __reset(_i, _bb); - } - - public Variant __assign(int _i, ByteBuffer _bb) { - __init(_i, _bb); - return this; - } - - public boolean nullable() { - int o = __offset(4); - return o != 0 ? 0 != bb.get(o + bb_pos) : false; - } - - public static final class Vector extends BaseVector { - public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { - __reset(_vector, _element_size, _bb); - return this; - } - - public Variant get(int j) { - return get(new Variant(), j); - } - - public Variant get(Variant obj, int j) { - return obj.__assign(__indirect(__element(j), bb), bb); - } - } -} - diff --git a/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsBuilder.java b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsBuilder.java new file mode 100644 index 000000000..8e53de8e5 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsBuilder.java @@ -0,0 +1,422 @@ +package io.github.dfa1.vortex.fbsrt; + +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_DOUBLE; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_FLOAT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_LONG; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +/// MemorySegment-native FlatBuffers builder — the write counterpart of [FbsTable]. +/// +/// Reimplements the standard FlatBuffers serialization algorithm (build back-to-front, +/// power-of-two field alignment, vtable construction with deduplication, default-value +/// omission) directly over an FFM [MemorySegment], with no `java.nio.ByteBuffer` and no +/// `com.google.flatbuffers` runtime. The emitted bytes are a valid FlatBuffer that the +/// Rust reference (and the flatc reader) can read. +/// +/// Usage mirrors the generated `createX` / `startX` / `addX` / `endX` helpers: scalars +/// and offsets are added in reverse field order between [#startTable(int)] and +/// [#endTable()]; vectors between [#startVector(int, int, int)] and [#endVector()]. +public final class FbsBuilder { + + private static final int SIZEOF_SHORT = 2; + private static final int SIZEOF_INT = 4; + + private MemorySegment buf; + private int space; + private int minalign = 1; + + private int[] vtable = new int[0]; + private int vtableInUse; + private int objectStart; + + private int[] vtables = new int[16]; + private int numVtables; + + private int vectorNumElems; + private boolean finished; + private boolean forceDefaults; + + /// Creates a builder with a small initial capacity that grows on demand. + public FbsBuilder() { + this(1024); + } + + /// Creates a builder with the given initial capacity. + /// + /// @param initialCapacity initial backing-buffer size in bytes (must be positive) + public FbsBuilder(int initialCapacity) { + this.buf = MemorySegment.ofArray(new byte[Math.max(1, initialCapacity)]); + this.space = (int) buf.byteSize(); + } + + private int capacity() { + return (int) buf.byteSize(); + } + + /// @return the current write position, measured from the end of the buffer + public int offset() { + return capacity() - space; + } + + private void growBuffer() { + int oldCap = capacity(); + if ((oldCap & 0xC0000000) != 0) { + throw new IllegalStateException("FlatBuffer too large"); + } + // The constructor guarantees capacity >= 1, so doubling always grows. + int newCap = oldCap << 1; + MemorySegment grown = MemorySegment.ofArray(new byte[newCap]); + MemorySegment.copy(buf, 0, grown, (long) newCap - oldCap, oldCap); + buf = grown; + space += newCap - oldCap; + } + + /// Aligns and reserves space for the next write. + /// + /// @param size alignment/size of the value about to be written (power of two) + /// @param additionalBytes extra bytes that will follow (e.g. a vector's payload) + public void prep(int size, int additionalBytes) { + if (size > minalign) { + minalign = size; + } + int alignSize = ((~(offset() + additionalBytes)) + 1) & (size - 1); + while (space < alignSize + size + additionalBytes) { + growBuffer(); + } + pad(alignSize); + } + + /// Writes `n` zero padding bytes (used by generated struct serializers to fill + /// inter-field and trailing gaps). + /// + /// @param n number of padding bytes + public void pad(int n) { + for (int i = 0; i < n; i++) { + buf.set(ValueLayout.JAVA_BYTE, --space, (byte) 0); + } + } + + /// @param x byte to write at the current position + public void putByte(byte x) { + buf.set(ValueLayout.JAVA_BYTE, space -= 1, x); + } + + /// @param x little-endian short to write + public void putShort(short x) { + buf.set(LE_SHORT, space -= SIZEOF_SHORT, x); + } + + /// @param x little-endian int to write + public void putInt(int x) { + buf.set(LE_INT, space -= SIZEOF_INT, x); + } + + /// @param x little-endian long to write + public void putLong(long x) { + buf.set(LE_LONG, space -= 8, x); + } + + /// @param x little-endian float to write + public void putFloat(float x) { + buf.set(LE_FLOAT, space -= SIZEOF_INT, x); + } + + /// @param x little-endian double to write + public void putDouble(double x) { + buf.set(LE_DOUBLE, space -= 8, x); + } + + /// @param x byte to add (aligned) + public void addByte(byte x) { + prep(1, 0); + putByte(x); + } + + /// @param x short to add (aligned) + public void addShort(short x) { + prep(SIZEOF_SHORT, 0); + putShort(x); + } + + /// @param x int to add (aligned) + public void addInt(int x) { + prep(SIZEOF_INT, 0); + putInt(x); + } + + /// @param x long to add (aligned) + public void addLong(long x) { + prep(8, 0); + putLong(x); + } + + /// @param x float to add (aligned) + public void addFloat(float x) { + prep(SIZEOF_INT, 0); + putFloat(x); + } + + /// @param x double to add (aligned) + public void addDouble(double x) { + prep(8, 0); + putDouble(x); + } + + /// Writes a relative offset (uoffset) to a previously written object. + /// + /// @param off absolute offset of the referenced object + public void addOffset(int off) { + prep(SIZEOF_INT, 0); + putInt(offset() - off + SIZEOF_INT); + } + + // ---- tables --------------------------------------------------------------- + + /// Begins a table with `numFields` vtable slots. + /// + /// @param numFields number of fields declared by the table + public void startTable(int numFields) { + if (vtable.length < numFields) { + vtable = new int[numFields]; + } else { + Arrays.fill(vtable, 0, numFields, 0); + } + vtableInUse = numFields; + objectStart = offset(); + } + + private void slot(int voffset) { + vtable[voffset] = offset(); + } + + /// When enabled, fields equal to their default are still written (instead of omitted), + /// e.g. to force-serialize a `null_count = 0` so readers can distinguish it from unknown. + /// + /// @param force whether to force-write default-valued fields + public void forceDefaults(boolean force) { + this.forceDefaults = force; + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addBoolean(int voffset, boolean x, boolean def) { + if (forceDefaults || x != def) { + addByte((byte) (x ? 1 : 0)); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addByte(int voffset, int x, int def) { + if (forceDefaults || x != def) { + addByte((byte) x); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addShort(int voffset, int x, int def) { + if (forceDefaults || x != def) { + addShort((short) x); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addInt(int voffset, int x, int def) { + if (forceDefaults || x != def) { + addInt(x); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addLong(int voffset, long x, long def) { + if (forceDefaults || x != def) { + addLong(x); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addFloat(int voffset, float x, float def) { + if (forceDefaults || x != def) { + addFloat(x); + slot(voffset); + } + } + + /// @param voffset field slot + /// @param x value + /// @param def default; the field is omitted when equal + public void addDouble(int voffset, double x, double def) { + if (forceDefaults || x != def) { + addDouble(x); + slot(voffset); + } + } + + /// Adds an offset field (to a table, vector or string). + /// + /// @param voffset field slot + /// @param x absolute offset of the referenced object + /// @param def default; the field is omitted when equal (0 = absent) + public void addOffset(int voffset, int x, int def) { + if (forceDefaults || x != def) { + addOffset(x); + slot(voffset); + } + } + + /// Adds an inline struct field (the struct must have just been written). + /// + /// @param voffset field slot + /// @param x offset of the inline struct (must equal the current offset) + /// @param def default; the field is omitted when equal + public void addStruct(int voffset, int x, int def) { + if (forceDefaults || x != def) { + if (x != offset()) { + throw new IllegalStateException("struct must be serialized inline"); + } + slot(voffset); + } + } + + /// Finishes the current table, emitting its vtable (deduplicated). + /// + /// @return the offset of the finished table + public int endTable() { + addInt(0); // soffset placeholder + int vtableloc = offset(); + + int i = vtableInUse - 1; + while (i >= 0 && vtable[i] == 0) { + i--; + } + int trimmedSize = i + 1; + for (; i >= 0; i--) { + addShort((short) (vtable[i] != 0 ? vtableloc - vtable[i] : 0)); + } + addShort((short) (vtableloc - objectStart)); + addShort((short) ((trimmedSize + 2) * SIZEOF_SHORT)); + + int cap = capacity(); + int existing = 0; + outer: + for (int j = 0; j < numVtables; j++) { + int vt1 = cap - vtables[j]; + int vt2 = space; + int len = buf.get(LE_SHORT, vt1) & 0xFFFF; + if (len != (buf.get(LE_SHORT, vt2) & 0xFFFF)) { + continue; + } + for (int k = SIZEOF_SHORT; k < len; k += SIZEOF_SHORT) { + if (buf.get(LE_SHORT, vt1 + k) != buf.get(LE_SHORT, vt2 + k)) { + continue outer; + } + } + existing = vtables[j]; + break; + } + + if (existing != 0) { + space = cap - vtableloc; + buf.set(LE_INT, space, existing - vtableloc); + } else { + if (numVtables == vtables.length) { + vtables = Arrays.copyOf(vtables, numVtables * 2); + } + vtables[numVtables++] = offset(); + buf.set(LE_INT, cap - vtableloc, offset() - vtableloc); + } + return vtableloc; + } + + // ---- vectors & strings ---------------------------------------------------- + + /// Begins a vector. Elements are added in reverse order, then [#endVector()] called. + /// + /// @param elemSize byte size of one element + /// @param numElems number of elements + /// @param alignment element alignment + public void startVector(int elemSize, int numElems, int alignment) { + vectorNumElems = numElems; + prep(SIZEOF_INT, elemSize * numElems); + prep(alignment, elemSize * numElems); + } + + /// Finishes the current vector. + /// + /// @return the offset of the finished vector + public int endVector() { + putInt(vectorNumElems); + return offset(); + } + + /// Writes a byte vector in one shot. + /// + /// @param data the bytes + /// @return the offset of the finished vector + public int createByteVector(byte[] data) { + int length = data.length; + startVector(1, length, 1); + space -= length; + MemorySegment.copy(data, 0, buf, ValueLayout.JAVA_BYTE, space, length); + return endVector(); + } + + /// Writes a length-prefixed, null-terminated UTF-8 string. + /// + /// @param s the string + /// @return the offset of the finished string + public int createString(String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + addByte((byte) 0); + int length = bytes.length; + startVector(1, length, 1); + space -= length; + MemorySegment.copy(bytes, 0, buf, ValueLayout.JAVA_BYTE, space, length); + return endVector(); + } + + // ---- finishing ------------------------------------------------------------ + + /// Finishes the buffer with the given root table. + /// + /// @param rootTable offset of the root table + public void finish(int rootTable) { + prep(minalign, SIZEOF_INT); + addOffset(rootTable); + finished = true; + } + + /// @return a zero-copy slice covering exactly the finished FlatBuffer + public MemorySegment dataSegment() { + if (!finished) { + throw new IllegalStateException("buffer not finished"); + } + return buf.asSlice(space, (long) capacity() - space); + } + + /// @return a copy of the finished FlatBuffer as a byte array + public byte[] sizedByteArray() { + return dataSegment().toArray(ValueLayout.JAVA_BYTE); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsStruct.java b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsStruct.java new file mode 100644 index 000000000..22553f6ff --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsStruct.java @@ -0,0 +1,74 @@ +package io.github.dfa1.vortex.fbsrt; + +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_DOUBLE; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_FLOAT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_LONG; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/// MemorySegment-native base for generated FlatBuffers struct accessors. +/// +/// Unlike a table, a FlatBuffers struct is a fixed-size, inline record with no +/// vtable: every field lives at a compile-time-constant byte offset from the +/// struct position, and every field is always present. Generated accessors read +/// directly at `position + fieldOffset`. +public class FbsStruct { + + /// The backing buffer. + protected MemorySegment seg; + + /// This struct's byte position within [#seg]. + protected long pos; + + /// Creates an unpositioned cursor; call [#init(MemorySegment, long)] before use. + protected FbsStruct() { + } + + /// Positions this cursor at a struct. + /// + /// @param segment backing buffer + /// @param position struct position within the buffer + protected final void init(MemorySegment segment, long position) { + this.seg = segment; + this.pos = position; + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the signed byte field + protected final byte readByte(int fieldOffset) { + return seg.get(ValueLayout.JAVA_BYTE, pos + fieldOffset); + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the little-endian 16-bit field + protected final short readShort(int fieldOffset) { + return seg.get(LE_SHORT, pos + fieldOffset); + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the little-endian 32-bit field + protected final int readInt(int fieldOffset) { + return seg.get(LE_INT, pos + fieldOffset); + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the little-endian 64-bit field + protected final long readLong(int fieldOffset) { + return seg.get(LE_LONG, pos + fieldOffset); + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the little-endian 32-bit float field + protected final float readFloat(int fieldOffset) { + return seg.get(LE_FLOAT, pos + fieldOffset); + } + + /// @param fieldOffset constant byte offset of the field within the struct + /// @return the little-endian 64-bit double field + protected final double readDouble(int fieldOffset) { + return seg.get(LE_DOUBLE, pos + fieldOffset); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsTable.java b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsTable.java new file mode 100644 index 000000000..2e6d2cd12 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/fbsrt/FbsTable.java @@ -0,0 +1,179 @@ +package io.github.dfa1.vortex.fbsrt; + +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_DOUBLE; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_FLOAT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_LONG; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +/// MemorySegment-native base for generated FlatBuffers table accessors. +/// +/// A FlatBuffers table is addressed by a position into a little-endian buffer. +/// The first 4 bytes at that position are a signed offset (`soffset`) back to the +/// table's vtable; the vtable maps each field's index to a 16-bit offset relative +/// to the table position (0 = field absent, read its default). This base provides +/// the small set of primitives the generated code needs to navigate that layout +/// over an FFM [MemorySegment] instead of a `java.nio.ByteBuffer`. +/// +/// Instances are mutable cursors (position + segment) reused during a scan, mirroring +/// the flatc-generated `__assign` pattern: zero allocation on the hot read path. +/// +/// Bounds: these accessors do not validate offsets or lengths read from the buffer +/// (e.g. [#readStringAt(long)] allocates `new byte[len]` with `len` taken straight from +/// the blob). This matches the previous `com.google.flatbuffers` runtime — the caller is +/// expected to have framed the FlatBuffer from a trusted region. File-level framing +/// (segment offsets/lengths, trailer, postscript) is range-checked upstream by the +/// reader's `IoBounds`; intra-FlatBuffer fields are not. +public class FbsTable { + + /// The backing buffer (typically a zero-copy slice of the mmap'd file). + protected MemorySegment seg; + + /// This table's byte position within [#seg]. + protected long pos; + + /// Creates an unpositioned cursor; call [#init(MemorySegment, long)] before use. + protected FbsTable() { + } + + /// Positions this cursor at a table. + /// + /// @param segment backing buffer + /// @param position table position within the buffer + protected final void init(MemorySegment segment, long position) { + this.seg = segment; + this.pos = position; + } + + /// Resolves the root table position of a finished FlatBuffer. + /// + /// @param segment buffer whose first 4 bytes hold the uoffset to the root table + /// @param start byte position of the buffer start within `segment` + /// @return the absolute position of the root table + public static long rootPosition(MemorySegment segment, long start) { + return start + segment.get(LE_INT, start); + } + + /// Looks up a field's offset relative to this table. + /// + /// @param vtableOffset the field's vtable slot (`4 + fieldIndex * 2`) + /// @return the field's offset relative to the table position, or 0 if the field + /// is absent (caller falls back to the field default) + protected final int fieldOffset(int vtableOffset) { + long vtable = pos - seg.get(LE_INT, pos); + int vtableSize = seg.get(LE_SHORT, vtable) & 0xFFFF; + return vtableOffset < vtableSize ? (seg.get(LE_SHORT, vtable + vtableOffset) & 0xFFFF) : 0; + } + + /// Follows a `uoffset` stored at an absolute position. + /// + /// @param at absolute position of the 32-bit uoffset + /// @return the absolute position it points to + protected final long indirect(long at) { + return at + seg.get(LE_INT, at); + } + + /// Resolves the absolute position of a vector's first element. + /// + /// @param fieldOffset the field offset returned by [#fieldOffset(int)] (non-zero) + /// @return absolute position of element 0 (past the 32-bit length prefix) + protected final long vectorElements(int fieldOffset) { + long at = fieldOffset + pos; + return at + seg.get(LE_INT, at) + 4; + } + + /// Reads a vector's element count. + /// + /// @param fieldOffset the field offset returned by [#fieldOffset(int)] (non-zero) + /// @return number of elements + protected final int vectorLength(int fieldOffset) { + long at = fieldOffset + pos; + at += seg.get(LE_INT, at); + return seg.get(LE_INT, at); + } + + /// Resolves the position of a union's selected member table. + /// + /// @param fieldOffset the union value field offset (non-zero) + /// @return absolute position of the member table + protected final long unionMemberPosition(int fieldOffset) { + long at = fieldOffset + pos; + return at + seg.get(LE_INT, at); + } + + /// Positions another table cursor onto this table's buffer (used by generated union + /// accessors to project the selected member table without exposing the buffer). + /// + /// @param other the cursor to position + /// @param position the table position + /// @param the cursor type + /// @return `other`, positioned + protected final T locate(T other, long position) { + other.seg = this.seg; + other.pos = position; + return other; + } + + /// Reads a length-prefixed UTF-8 string whose `uoffset` lives at an absolute position. + /// Use `readStringAt(pos + fieldOffset)` for a string field and + /// `readStringAt(vectorElements(o) + j * 4)` for the j-th element of a string vector. + /// + /// @param uoffsetAt absolute position of the 32-bit offset to the string + /// @return the decoded string + protected final String readStringAt(long uoffsetAt) { + long start = uoffsetAt + seg.get(LE_INT, uoffsetAt); + int len = seg.get(LE_INT, start); + byte[] bytes = new byte[len]; + MemorySegment.copy(seg, ValueLayout.JAVA_BYTE, start + 4, bytes, 0, len); + return new String(bytes, StandardCharsets.UTF_8); + } + + /// Returns a zero-copy slice of a byte/ubyte vector field. + /// + /// @param fieldOffset the field offset returned by [#fieldOffset(int)] (non-zero) + /// @return a slice covering exactly the vector's bytes + protected final MemorySegment byteVector(int fieldOffset) { + long elements = vectorElements(fieldOffset); + return seg.asSlice(elements, vectorLength(fieldOffset)); + } + + /// @param at absolute position + /// @return the signed byte at `at` + protected final byte readByte(long at) { + return seg.get(ValueLayout.JAVA_BYTE, at); + } + + /// @param at absolute position + /// @return the little-endian 16-bit value at `at` + protected final short readShort(long at) { + return seg.get(LE_SHORT, at); + } + + /// @param at absolute position + /// @return the little-endian 32-bit value at `at` + protected final int readInt(long at) { + return seg.get(LE_INT, at); + } + + /// @param at absolute position + /// @return the little-endian 64-bit value at `at` + protected final long readLong(long at) { + return seg.get(LE_LONG, at); + } + + /// @param at absolute position + /// @return the little-endian 32-bit float at `at` + protected final float readFloat(long at) { + return seg.get(LE_FLOAT, at); + } + + /// @param at absolute position + /// @return the little-endian 64-bit double at `at` + protected final double readDouble(long at) { + return seg.get(LE_DOUBLE, at); + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ALPMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPMetadata.java similarity index 84% rename from core/src/main/java/io/github/dfa1/vortex/proto/ALPMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPMetadata.java index 4792964d2..3fdcbf585 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ALPMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPMetadata.java @@ -10,10 +10,10 @@ /// @param exp_f field tag 2 /// @param patches field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ALPMetadata( +public record ProtoALPMetadata( int exp_e, int exp_f, - PatchesMetadata patches + ProtoPatchesMetadata patches ) { /// Decodes a {@code vortex.encodings.ALPMetadata} from a slice of a memory segment. @@ -22,11 +22,11 @@ public record ALPMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ALPMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoALPMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int exp_e = 0; int exp_f = 0; - PatchesMetadata patches = null; + ProtoPatchesMetadata patches = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -38,12 +38,12 @@ public static ALPMetadata decode(MemorySegment __seg, long __off, long __len) th } case 3 -> { MemorySegment __slice = r.readLenDelimSegment(); - patches = PatchesMetadata.decode(__slice, 0, __slice.byteSize()); + patches = ProtoPatchesMetadata.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new ALPMetadata(exp_e, exp_f, patches); + return new ProtoALPMetadata(exp_e, exp_f, patches); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ALPRDMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPRDMetadata.java similarity index 83% rename from core/src/main/java/io/github/dfa1/vortex/proto/ALPRDMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPRDMetadata.java index b5b8d1a11..68591da65 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ALPRDMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoALPRDMetadata.java @@ -12,12 +12,12 @@ /// @param left_parts_ptype field tag 4 /// @param patches field tag 5 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ALPRDMetadata( +public record ProtoALPRDMetadata( int right_bit_width, int dict_len, java.util.List dict, - PType left_parts_ptype, - PatchesMetadata patches + ProtoPType left_parts_ptype, + ProtoPatchesMetadata patches ) { /// Decodes a {@code vortex.encodings.ALPRDMetadata} from a slice of a memory segment. @@ -26,13 +26,13 @@ public record ALPRDMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ALPRDMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoALPRDMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int right_bit_width = 0; int dict_len = 0; java.util.List dict = new java.util.ArrayList<>(); - PType left_parts_ptype = PType.U8; - PatchesMetadata patches = null; + ProtoPType left_parts_ptype = ProtoPType.U8; + ProtoPatchesMetadata patches = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -55,19 +55,19 @@ public static ALPRDMetadata decode(MemorySegment __seg, long __off, long __len) case 4 -> { int __ev = r.readVarint32(); try { - left_parts_ptype = PType.fromValue(__ev); + left_parts_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 5 -> { MemorySegment __slice = r.readLenDelimSegment(); - patches = PatchesMetadata.decode(__slice, 0, __slice.byteSize()); + patches = ProtoPatchesMetadata.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new ALPRDMetadata(right_bit_width, dict_len, dict, left_parts_ptype, patches); + return new ProtoALPRDMetadata(right_bit_width, dict_len, dict, left_parts_ptype, patches); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Binary.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBinary.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/Binary.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoBinary.java index b7a8b7cd5..d905fd1bd 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Binary.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBinary.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param nullable field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Binary( +public record ProtoBinary( boolean nullable ) { @@ -18,7 +18,7 @@ public record Binary( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Binary decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoBinary decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); boolean nullable = false; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Binary decode(MemorySegment __seg, long __off, long __len) throws default -> r.skipField(tag & 7); } } - return new Binary(nullable); + return new ProtoBinary(nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/BitPackedMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBitPackedMetadata.java similarity index 84% rename from core/src/main/java/io/github/dfa1/vortex/proto/BitPackedMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoBitPackedMetadata.java index 4dae8820a..1af965d42 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/BitPackedMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBitPackedMetadata.java @@ -10,10 +10,10 @@ /// @param offset field tag 2 /// @param patches field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record BitPackedMetadata( +public record ProtoBitPackedMetadata( int bit_width, int offset, - PatchesMetadata patches + ProtoPatchesMetadata patches ) { /// Decodes a {@code vortex.encodings.BitPackedMetadata} from a slice of a memory segment. @@ -22,11 +22,11 @@ public record BitPackedMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static BitPackedMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoBitPackedMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int bit_width = 0; int offset = 0; - PatchesMetadata patches = null; + ProtoPatchesMetadata patches = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -38,12 +38,12 @@ public static BitPackedMetadata decode(MemorySegment __seg, long __off, long __l } case 3 -> { MemorySegment __slice = r.readLenDelimSegment(); - patches = PatchesMetadata.decode(__slice, 0, __slice.byteSize()); + patches = ProtoPatchesMetadata.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new BitPackedMetadata(bit_width, offset, patches); + return new ProtoBitPackedMetadata(bit_width, offset, patches); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Bool.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBool.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/Bool.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoBool.java index f27668df2..65be8db37 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Bool.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoBool.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param nullable field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Bool( +public record ProtoBool( boolean nullable ) { @@ -18,7 +18,7 @@ public record Bool( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Bool decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoBool decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); boolean nullable = false; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Bool decode(MemorySegment __seg, long __off, long __len) throws IO default -> r.skipField(tag & 7); } } - return new Bool(nullable); + return new ProtoBool(nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DType.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDType.java similarity index 64% rename from core/src/main/java/io/github/dfa1/vortex/proto/DType.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDType.java index a009b535d..cd87206cc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DType.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDType.java @@ -19,19 +19,19 @@ /// @param variant field tag 11 /// @param union field tag 12 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DType( - Null null_, - Bool bool, - Primitive primitive, - Decimal decimal, - Utf8 utf8, - Binary binary, - Struct struct, - List list, - Extension extension, - FixedSizeList fixed_size_list, - Variant variant, - Union union +public record ProtoDType( + ProtoNull null_, + ProtoBool bool, + ProtoPrimitive primitive, + ProtoDecimal decimal, + ProtoUtf8 utf8, + ProtoBinary binary, + ProtoStruct struct, + ProtoList list, + ProtoExtension extension, + ProtoFixedSizeList fixed_size_list, + ProtoVariant variant, + ProtoUnion union ) { /// Decodes a {@code vortex.dtype.DType} from a slice of a memory segment. @@ -40,75 +40,75 @@ public record DType( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DType decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDType decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - Null null_ = null; - Bool bool = null; - Primitive primitive = null; - Decimal decimal = null; - Utf8 utf8 = null; - Binary binary = null; - Struct struct = null; - List list = null; - Extension extension = null; - FixedSizeList fixed_size_list = null; - Variant variant = null; - Union union = null; + ProtoNull null_ = null; + ProtoBool bool = null; + ProtoPrimitive primitive = null; + ProtoDecimal decimal = null; + ProtoUtf8 utf8 = null; + ProtoBinary binary = null; + ProtoStruct struct = null; + ProtoList list = null; + ProtoExtension extension = null; + ProtoFixedSizeList fixed_size_list = null; + ProtoVariant variant = null; + ProtoUnion union = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - null_ = Null.decode(__slice, 0, __slice.byteSize()); + null_ = ProtoNull.decode(__slice, 0, __slice.byteSize()); } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - bool = Bool.decode(__slice, 0, __slice.byteSize()); + bool = ProtoBool.decode(__slice, 0, __slice.byteSize()); } case 3 -> { MemorySegment __slice = r.readLenDelimSegment(); - primitive = Primitive.decode(__slice, 0, __slice.byteSize()); + primitive = ProtoPrimitive.decode(__slice, 0, __slice.byteSize()); } case 4 -> { MemorySegment __slice = r.readLenDelimSegment(); - decimal = Decimal.decode(__slice, 0, __slice.byteSize()); + decimal = ProtoDecimal.decode(__slice, 0, __slice.byteSize()); } case 5 -> { MemorySegment __slice = r.readLenDelimSegment(); - utf8 = Utf8.decode(__slice, 0, __slice.byteSize()); + utf8 = ProtoUtf8.decode(__slice, 0, __slice.byteSize()); } case 6 -> { MemorySegment __slice = r.readLenDelimSegment(); - binary = Binary.decode(__slice, 0, __slice.byteSize()); + binary = ProtoBinary.decode(__slice, 0, __slice.byteSize()); } case 7 -> { MemorySegment __slice = r.readLenDelimSegment(); - struct = Struct.decode(__slice, 0, __slice.byteSize()); + struct = ProtoStruct.decode(__slice, 0, __slice.byteSize()); } case 8 -> { MemorySegment __slice = r.readLenDelimSegment(); - list = List.decode(__slice, 0, __slice.byteSize()); + list = ProtoList.decode(__slice, 0, __slice.byteSize()); } case 9 -> { MemorySegment __slice = r.readLenDelimSegment(); - extension = Extension.decode(__slice, 0, __slice.byteSize()); + extension = ProtoExtension.decode(__slice, 0, __slice.byteSize()); } case 10 -> { MemorySegment __slice = r.readLenDelimSegment(); - fixed_size_list = FixedSizeList.decode(__slice, 0, __slice.byteSize()); + fixed_size_list = ProtoFixedSizeList.decode(__slice, 0, __slice.byteSize()); } case 11 -> { MemorySegment __slice = r.readLenDelimSegment(); - variant = Variant.decode(__slice, 0, __slice.byteSize()); + variant = ProtoVariant.decode(__slice, 0, __slice.byteSize()); } case 12 -> { MemorySegment __slice = r.readLenDelimSegment(); - union = Union.decode(__slice, 0, __slice.byteSize()); + union = ProtoUnion.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new DType(null_, bool, primitive, decimal, utf8, binary, struct, list, extension, fixed_size_list, variant, union); + return new ProtoDType(null_, bool, primitive, decimal, utf8, binary, struct, list, extension, fixed_size_list, variant, union); } /// Encodes this record to a proto3-wire-format byte array. @@ -197,84 +197,84 @@ void encodeTo(ProtoWriter w) { /// Factory for oneof case {@code null} (field tag 1). /// @param value the value to set /// @return a record with only the {@code null} component set - public static DType ofNull(Null value) { - return new DType(value, null, null, null, null, null, null, null, null, null, null, null); + public static ProtoDType ofNull(ProtoNull value) { + return new ProtoDType(value, null, null, null, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code bool} (field tag 2). /// @param value the value to set /// @return a record with only the {@code bool} component set - public static DType ofBool(Bool value) { - return new DType(null, value, null, null, null, null, null, null, null, null, null, null); + public static ProtoDType ofBool(ProtoBool value) { + return new ProtoDType(null, value, null, null, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code primitive} (field tag 3). /// @param value the value to set /// @return a record with only the {@code primitive} component set - public static DType ofPrimitive(Primitive value) { - return new DType(null, null, value, null, null, null, null, null, null, null, null, null); + public static ProtoDType ofPrimitive(ProtoPrimitive value) { + return new ProtoDType(null, null, value, null, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code decimal} (field tag 4). /// @param value the value to set /// @return a record with only the {@code decimal} component set - public static DType ofDecimal(Decimal value) { - return new DType(null, null, null, value, null, null, null, null, null, null, null, null); + public static ProtoDType ofDecimal(ProtoDecimal value) { + return new ProtoDType(null, null, null, value, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code utf8} (field tag 5). /// @param value the value to set /// @return a record with only the {@code utf8} component set - public static DType ofUtf8(Utf8 value) { - return new DType(null, null, null, null, value, null, null, null, null, null, null, null); + public static ProtoDType ofUtf8(ProtoUtf8 value) { + return new ProtoDType(null, null, null, null, value, null, null, null, null, null, null, null); } /// Factory for oneof case {@code binary} (field tag 6). /// @param value the value to set /// @return a record with only the {@code binary} component set - public static DType ofBinary(Binary value) { - return new DType(null, null, null, null, null, value, null, null, null, null, null, null); + public static ProtoDType ofBinary(ProtoBinary value) { + return new ProtoDType(null, null, null, null, null, value, null, null, null, null, null, null); } /// Factory for oneof case {@code struct} (field tag 7). /// @param value the value to set /// @return a record with only the {@code struct} component set - public static DType ofStruct(Struct value) { - return new DType(null, null, null, null, null, null, value, null, null, null, null, null); + public static ProtoDType ofStruct(ProtoStruct value) { + return new ProtoDType(null, null, null, null, null, null, value, null, null, null, null, null); } /// Factory for oneof case {@code list} (field tag 8). /// @param value the value to set /// @return a record with only the {@code list} component set - public static DType ofList(List value) { - return new DType(null, null, null, null, null, null, null, value, null, null, null, null); + public static ProtoDType ofList(ProtoList value) { + return new ProtoDType(null, null, null, null, null, null, null, value, null, null, null, null); } /// Factory for oneof case {@code extension} (field tag 9). /// @param value the value to set /// @return a record with only the {@code extension} component set - public static DType ofExtension(Extension value) { - return new DType(null, null, null, null, null, null, null, null, value, null, null, null); + public static ProtoDType ofExtension(ProtoExtension value) { + return new ProtoDType(null, null, null, null, null, null, null, null, value, null, null, null); } /// Factory for oneof case {@code fixed_size_list} (field tag 10). /// @param value the value to set /// @return a record with only the {@code fixed_size_list} component set - public static DType ofFixedSizeList(FixedSizeList value) { - return new DType(null, null, null, null, null, null, null, null, null, value, null, null); + public static ProtoDType ofFixedSizeList(ProtoFixedSizeList value) { + return new ProtoDType(null, null, null, null, null, null, null, null, null, value, null, null); } /// Factory for oneof case {@code variant} (field tag 11). /// @param value the value to set /// @return a record with only the {@code variant} component set - public static DType ofVariant(Variant value) { - return new DType(null, null, null, null, null, null, null, null, null, null, value, null); + public static ProtoDType ofVariant(ProtoVariant value) { + return new ProtoDType(null, null, null, null, null, null, null, null, null, null, value, null); } /// Factory for oneof case {@code union} (field tag 12). /// @param value the value to set /// @return a record with only the {@code union} component set - public static DType ofUnion(Union value) { - return new DType(null, null, null, null, null, null, null, null, null, null, null, value); + public static ProtoDType ofUnion(ProtoUnion value) { + return new ProtoDType(null, null, null, null, null, null, null, null, null, null, null, value); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DateTimePartsMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDateTimePartsMetadata.java similarity index 71% rename from core/src/main/java/io/github/dfa1/vortex/proto/DateTimePartsMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDateTimePartsMetadata.java index 680eb5043..e756dca0e 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DateTimePartsMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDateTimePartsMetadata.java @@ -10,10 +10,10 @@ /// @param seconds_ptype field tag 2 /// @param subseconds_ptype field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DateTimePartsMetadata( - PType days_ptype, - PType seconds_ptype, - PType subseconds_ptype +public record ProtoDateTimePartsMetadata( + ProtoPType days_ptype, + ProtoPType seconds_ptype, + ProtoPType subseconds_ptype ) { /// Decodes a {@code vortex.encodings.DateTimePartsMetadata} from a slice of a memory segment. @@ -22,42 +22,42 @@ public record DateTimePartsMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DateTimePartsMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDateTimePartsMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType days_ptype = PType.U8; - PType seconds_ptype = PType.U8; - PType subseconds_ptype = PType.U8; + ProtoPType days_ptype = ProtoPType.U8; + ProtoPType seconds_ptype = ProtoPType.U8; + ProtoPType subseconds_ptype = ProtoPType.U8; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { int __ev = r.readVarint32(); try { - days_ptype = PType.fromValue(__ev); + days_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 2 -> { int __ev = r.readVarint32(); try { - seconds_ptype = PType.fromValue(__ev); + seconds_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 3 -> { int __ev = r.readVarint32(); try { - subseconds_ptype = PType.fromValue(__ev); + subseconds_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } default -> r.skipField(tag & 7); } } - return new DateTimePartsMetadata(days_ptype, seconds_ptype, subseconds_ptype); + return new ProtoDateTimePartsMetadata(days_ptype, seconds_ptype, subseconds_ptype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Decimal.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimal.java similarity index 91% rename from core/src/main/java/io/github/dfa1/vortex/proto/Decimal.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimal.java index 3143b1318..e171d8f3e 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Decimal.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimal.java @@ -10,7 +10,7 @@ /// @param scale field tag 2 /// @param nullable field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Decimal( +public record ProtoDecimal( int precision, int scale, boolean nullable @@ -22,7 +22,7 @@ public record Decimal( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Decimal decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDecimal decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int precision = 0; int scale = 0; @@ -42,7 +42,7 @@ public static Decimal decode(MemorySegment __seg, long __off, long __len) throws default -> r.skipField(tag & 7); } } - return new Decimal(precision, scale, nullable); + return new ProtoDecimal(precision, scale, nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DecimalBytePartsMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalBytePartsMetadata.java similarity index 79% rename from core/src/main/java/io/github/dfa1/vortex/proto/DecimalBytePartsMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalBytePartsMetadata.java index b780fdd55..d83576849 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DecimalBytePartsMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalBytePartsMetadata.java @@ -9,8 +9,8 @@ /// @param zeroth_child_ptype field tag 1 /// @param lower_part_count field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DecimalBytePartsMetadata( - PType zeroth_child_ptype, +public record ProtoDecimalBytePartsMetadata( + ProtoPType zeroth_child_ptype, int lower_part_count ) { @@ -20,9 +20,9 @@ public record DecimalBytePartsMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DecimalBytePartsMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDecimalBytePartsMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType zeroth_child_ptype = PType.U8; + ProtoPType zeroth_child_ptype = ProtoPType.U8; int lower_part_count = 0; while (r.hasMore()) { int tag = r.readVarint32(); @@ -30,9 +30,9 @@ public static DecimalBytePartsMetadata decode(MemorySegment __seg, long __off, l case 1 -> { int __ev = r.readVarint32(); try { - zeroth_child_ptype = PType.fromValue(__ev); + zeroth_child_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 2 -> { @@ -41,7 +41,7 @@ public static DecimalBytePartsMetadata decode(MemorySegment __seg, long __off, l default -> r.skipField(tag & 7); } } - return new DecimalBytePartsMetadata(zeroth_child_ptype, lower_part_count); + return new ProtoDecimalBytePartsMetadata(zeroth_child_ptype, lower_part_count); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DecimalMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalMetadata.java similarity index 88% rename from core/src/main/java/io/github/dfa1/vortex/proto/DecimalMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalMetadata.java index 64fc8728e..dce8e84be 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DecimalMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDecimalMetadata.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param values_type field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DecimalMetadata( +public record ProtoDecimalMetadata( int values_type ) { @@ -18,7 +18,7 @@ public record DecimalMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DecimalMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDecimalMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int values_type = 0; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static DecimalMetadata decode(MemorySegment __seg, long __off, long __len default -> r.skipField(tag & 7); } } - return new DecimalMetadata(values_type); + return new ProtoDecimalMetadata(values_type); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DeltaMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDeltaMetadata.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/DeltaMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDeltaMetadata.java index b88835059..0c55012ef 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DeltaMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDeltaMetadata.java @@ -9,7 +9,7 @@ /// @param deltas_len field tag 1 /// @param offset field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DeltaMetadata( +public record ProtoDeltaMetadata( long deltas_len, int offset ) { @@ -20,7 +20,7 @@ public record DeltaMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DeltaMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDeltaMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long deltas_len = 0; int offset = 0; @@ -36,7 +36,7 @@ public static DeltaMetadata decode(MemorySegment __seg, long __off, long __len) default -> r.skipField(tag & 7); } } - return new DeltaMetadata(deltas_len, offset); + return new ProtoDeltaMetadata(deltas_len, offset); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/DictMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDictMetadata.java similarity index 84% rename from core/src/main/java/io/github/dfa1/vortex/proto/DictMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoDictMetadata.java index 0acef874e..5df00536f 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/DictMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoDictMetadata.java @@ -11,9 +11,9 @@ /// @param is_nullable_codes field tag 3 /// @param all_values_referenced field tag 4 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record DictMetadata( +public record ProtoDictMetadata( int values_len, - PType codes_ptype, + ProtoPType codes_ptype, Boolean is_nullable_codes, Boolean all_values_referenced ) { @@ -24,10 +24,10 @@ public record DictMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static DictMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoDictMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int values_len = 0; - PType codes_ptype = PType.U8; + ProtoPType codes_ptype = ProtoPType.U8; Boolean is_nullable_codes = null; Boolean all_values_referenced = null; while (r.hasMore()) { @@ -39,9 +39,9 @@ public static DictMetadata decode(MemorySegment __seg, long __off, long __len) t case 2 -> { int __ev = r.readVarint32(); try { - codes_ptype = PType.fromValue(__ev); + codes_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 3 -> { @@ -53,7 +53,7 @@ public static DictMetadata decode(MemorySegment __seg, long __off, long __len) t default -> r.skipField(tag & 7); } } - return new DictMetadata(values_len, codes_ptype, is_nullable_codes, all_values_referenced); + return new ProtoDictMetadata(values_len, codes_ptype, is_nullable_codes, all_values_referenced); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Extension.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoExtension.java similarity index 86% rename from core/src/main/java/io/github/dfa1/vortex/proto/Extension.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoExtension.java index 14121c1d7..2226917f2 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Extension.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoExtension.java @@ -11,9 +11,9 @@ /// @param storage_dtype field tag 2 /// @param metadata field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Extension( +public record ProtoExtension( String id, - DType storage_dtype, + ProtoDType storage_dtype, byte[] metadata ) { @@ -23,10 +23,10 @@ public record Extension( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Extension decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoExtension decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); String id = ""; - DType storage_dtype = null; + ProtoDType storage_dtype = null; byte[] metadata = null; while (r.hasMore()) { int tag = r.readVarint32(); @@ -36,7 +36,7 @@ public static Extension decode(MemorySegment __seg, long __off, long __len) thro } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - storage_dtype = DType.decode(__slice, 0, __slice.byteSize()); + storage_dtype = ProtoDType.decode(__slice, 0, __slice.byteSize()); } case 3 -> { metadata = r.readBytes(); @@ -44,7 +44,7 @@ public static Extension decode(MemorySegment __seg, long __off, long __len) thro default -> r.skipField(tag & 7); } } - return new Extension(id, storage_dtype, metadata); + return new ProtoExtension(id, storage_dtype, metadata); } /// Encodes this record to a proto3-wire-format byte array. @@ -77,7 +77,7 @@ public boolean equals(Object __o) { if (this == __o) { return true; } - if (!(__o instanceof Extension __that)) { + if (!(__o instanceof ProtoExtension __that)) { return false; } return java.util.Objects.equals(id, __that.id) diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/FSSTMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFSSTMetadata.java similarity index 72% rename from core/src/main/java/io/github/dfa1/vortex/proto/FSSTMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoFSSTMetadata.java index 4137da41c..57f4be3e0 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/FSSTMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFSSTMetadata.java @@ -9,9 +9,9 @@ /// @param uncompressed_lengths_ptype field tag 1 /// @param codes_offsets_ptype field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record FSSTMetadata( - PType uncompressed_lengths_ptype, - PType codes_offsets_ptype +public record ProtoFSSTMetadata( + ProtoPType uncompressed_lengths_ptype, + ProtoPType codes_offsets_ptype ) { /// Decodes a {@code vortex.encodings.FSSTMetadata} from a slice of a memory segment. @@ -20,33 +20,33 @@ public record FSSTMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static FSSTMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoFSSTMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType uncompressed_lengths_ptype = PType.U8; - PType codes_offsets_ptype = PType.U8; + ProtoPType uncompressed_lengths_ptype = ProtoPType.U8; + ProtoPType codes_offsets_ptype = ProtoPType.U8; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { int __ev = r.readVarint32(); try { - uncompressed_lengths_ptype = PType.fromValue(__ev); + uncompressed_lengths_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 2 -> { int __ev = r.readVarint32(); try { - codes_offsets_ptype = PType.fromValue(__ev); + codes_offsets_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } default -> r.skipField(tag & 7); } } - return new FSSTMetadata(uncompressed_lengths_ptype, codes_offsets_ptype); + return new ProtoFSSTMetadata(uncompressed_lengths_ptype, codes_offsets_ptype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Field.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoField.java similarity index 86% rename from core/src/main/java/io/github/dfa1/vortex/proto/Field.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoField.java index 33f131cbb..ac257936b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Field.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoField.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param name field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Field( +public record ProtoField( String name ) { @@ -18,7 +18,7 @@ public record Field( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Field decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoField decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); String name = null; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Field decode(MemorySegment __seg, long __off, long __len) throws I default -> r.skipField(tag & 7); } } - return new Field(name); + return new ProtoField(name); } /// Encodes this record to a proto3-wire-format byte array. @@ -51,7 +51,7 @@ void encodeTo(ProtoWriter w) { /// Factory for oneof case {@code name} (field tag 1). /// @param value the value to set /// @return a record with only the {@code name} component set - public static Field ofName(String value) { - return new Field(value); + public static ProtoField ofName(String value) { + return new ProtoField(value); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/FieldPath.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFieldPath.java similarity index 78% rename from core/src/main/java/io/github/dfa1/vortex/proto/FieldPath.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoFieldPath.java index 20aa208ac..5589cb2c9 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/FieldPath.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFieldPath.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param path field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record FieldPath( - java.util.List path +public record ProtoFieldPath( + java.util.List path ) { /// Decodes a {@code vortex.dtype.FieldPath} from a slice of a memory segment. @@ -18,20 +18,20 @@ public record FieldPath( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static FieldPath decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoFieldPath decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - java.util.List path = new java.util.ArrayList<>(); + java.util.List path = new java.util.ArrayList<>(); while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - path.add(Field.decode(__slice, 0, __slice.byteSize())); + path.add(ProtoField.decode(__slice, 0, __slice.byteSize())); } default -> r.skipField(tag & 7); } } - return new FieldPath(path); + return new ProtoFieldPath(path); } /// Encodes this record to a proto3-wire-format byte array. @@ -43,7 +43,7 @@ public byte[] encode() { } void encodeTo(ProtoWriter w) { - for (Field __v : path) { + for (ProtoField __v : path) { w.writeTag(1, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/FixedSizeList.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFixedSizeList.java similarity index 84% rename from core/src/main/java/io/github/dfa1/vortex/proto/FixedSizeList.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoFixedSizeList.java index 9d0a7fb66..420b04252 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/FixedSizeList.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoFixedSizeList.java @@ -10,8 +10,8 @@ /// @param size field tag 2 /// @param nullable field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record FixedSizeList( - DType element_type, +public record ProtoFixedSizeList( + ProtoDType element_type, int size, boolean nullable ) { @@ -22,9 +22,9 @@ public record FixedSizeList( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static FixedSizeList decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoFixedSizeList decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - DType element_type = null; + ProtoDType element_type = null; int size = 0; boolean nullable = false; while (r.hasMore()) { @@ -32,7 +32,7 @@ public static FixedSizeList decode(MemorySegment __seg, long __off, long __len) switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - element_type = DType.decode(__slice, 0, __slice.byteSize()); + element_type = ProtoDType.decode(__slice, 0, __slice.byteSize()); } case 2 -> { size = r.readVarint32(); @@ -43,7 +43,7 @@ public static FixedSizeList decode(MemorySegment __seg, long __off, long __len) default -> r.skipField(tag & 7); } } - return new FixedSizeList(element_type, size, nullable); + return new ProtoFixedSizeList(element_type, size, nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/List.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoList.java similarity index 84% rename from core/src/main/java/io/github/dfa1/vortex/proto/List.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoList.java index ad9872096..803b33912 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/List.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoList.java @@ -9,8 +9,8 @@ /// @param element_type field tag 1 /// @param nullable field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record List( - DType element_type, +public record ProtoList( + ProtoDType element_type, boolean nullable ) { @@ -20,16 +20,16 @@ public record List( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static List decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoList decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - DType element_type = null; + ProtoDType element_type = null; boolean nullable = false; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - element_type = DType.decode(__slice, 0, __slice.byteSize()); + element_type = ProtoDType.decode(__slice, 0, __slice.byteSize()); } case 2 -> { nullable = r.readBool(); @@ -37,7 +37,7 @@ public static List decode(MemorySegment __seg, long __off, long __len) throws IO default -> r.skipField(tag & 7); } } - return new List(element_type, nullable); + return new ProtoList(element_type, nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ListMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListMetadata.java similarity index 81% rename from core/src/main/java/io/github/dfa1/vortex/proto/ListMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoListMetadata.java index 43d09c2c5..d84aac436 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ListMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListMetadata.java @@ -9,9 +9,9 @@ /// @param elements_len field tag 1 /// @param offset_ptype field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ListMetadata( +public record ProtoListMetadata( long elements_len, - PType offset_ptype + ProtoPType offset_ptype ) { /// Decodes a {@code vortex.encodings.ListMetadata} from a slice of a memory segment. @@ -20,10 +20,10 @@ public record ListMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ListMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoListMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long elements_len = 0; - PType offset_ptype = PType.U8; + ProtoPType offset_ptype = ProtoPType.U8; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -33,15 +33,15 @@ public static ListMetadata decode(MemorySegment __seg, long __off, long __len) t case 2 -> { int __ev = r.readVarint32(); try { - offset_ptype = PType.fromValue(__ev); + offset_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } default -> r.skipField(tag & 7); } } - return new ListMetadata(elements_len, offset_ptype); + return new ProtoListMetadata(elements_len, offset_ptype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ListValue.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListValue.java similarity index 77% rename from core/src/main/java/io/github/dfa1/vortex/proto/ListValue.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoListValue.java index 989525166..c5fdbea66 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ListValue.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListValue.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param values field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ListValue( - java.util.List values +public record ProtoListValue( + java.util.List values ) { /// Decodes a {@code vortex.scalar.ListValue} from a slice of a memory segment. @@ -18,20 +18,20 @@ public record ListValue( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ListValue decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoListValue decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - java.util.List values = new java.util.ArrayList<>(); + java.util.List values = new java.util.ArrayList<>(); while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - values.add(ScalarValue.decode(__slice, 0, __slice.byteSize())); + values.add(ProtoScalarValue.decode(__slice, 0, __slice.byteSize())); } default -> r.skipField(tag & 7); } } - return new ListValue(values); + return new ProtoListValue(values); } /// Encodes this record to a proto3-wire-format byte array. @@ -43,7 +43,7 @@ public byte[] encode() { } void encodeTo(ProtoWriter w) { - for (ScalarValue __v : values) { + for (ProtoScalarValue __v : values) { w.writeTag(1, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ListViewMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListViewMetadata.java similarity index 76% rename from core/src/main/java/io/github/dfa1/vortex/proto/ListViewMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoListViewMetadata.java index 7b9148b56..d0a338339 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ListViewMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoListViewMetadata.java @@ -10,10 +10,10 @@ /// @param offset_ptype field tag 2 /// @param size_ptype field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ListViewMetadata( +public record ProtoListViewMetadata( long elements_len, - PType offset_ptype, - PType size_ptype + ProtoPType offset_ptype, + ProtoPType size_ptype ) { /// Decodes a {@code vortex.encodings.ListViewMetadata} from a slice of a memory segment. @@ -22,11 +22,11 @@ public record ListViewMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ListViewMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoListViewMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long elements_len = 0; - PType offset_ptype = PType.U8; - PType size_ptype = PType.U8; + ProtoPType offset_ptype = ProtoPType.U8; + ProtoPType size_ptype = ProtoPType.U8; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -36,23 +36,23 @@ public static ListViewMetadata decode(MemorySegment __seg, long __off, long __le case 2 -> { int __ev = r.readVarint32(); try { - offset_ptype = PType.fromValue(__ev); + offset_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 3 -> { int __ev = r.readVarint32(); try { - size_ptype = PType.fromValue(__ev); + size_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } default -> r.skipField(tag & 7); } } - return new ListViewMetadata(elements_len, offset_ptype, size_ptype); + return new ProtoListViewMetadata(elements_len, offset_ptype, size_ptype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Null.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoNull.java similarity index 88% rename from core/src/main/java/io/github/dfa1/vortex/proto/Null.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoNull.java index fd2a83603..6d00c19dc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Null.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoNull.java @@ -7,7 +7,7 @@ /// Generated from proto3 message {@code vortex.dtype.Null}. /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Null( +public record ProtoNull( ) { /// Decodes a {@code vortex.dtype.Null} from a slice of a memory segment. @@ -16,7 +16,7 @@ public record Null( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Null decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoNull decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); while (r.hasMore()) { int tag = r.readVarint32(); @@ -24,7 +24,7 @@ public static Null decode(MemorySegment __seg, long __off, long __len) throws IO default -> r.skipField(tag & 7); } } - return new Null(); + return new ProtoNull(); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/NullValue.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoNullValue.java similarity index 76% rename from core/src/main/java/io/github/dfa1/vortex/proto/NullValue.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoNullValue.java index 90643c600..81c41e640 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/NullValue.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoNullValue.java @@ -5,12 +5,12 @@ /// Generated from proto3 enum {@code google.protobuf.NullValue}. /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public enum NullValue { +public enum ProtoNullValue { NULL_VALUE(0); private final int value; - NullValue(int value) { + ProtoNullValue(int value) { this.value = value; } @@ -23,12 +23,12 @@ public int value() { /// @param value wire-format integer /// @return matching enum constant /// @throws IllegalArgumentException if no constant matches - public static NullValue fromValue(int value) { - for (NullValue v : values()) { + public static ProtoNullValue fromValue(int value) { + for (ProtoNullValue v : values()) { if (v.value == value) { return v; } } - throw new IllegalArgumentException("unknown NullValue value: " + value); + throw new IllegalArgumentException("unknown ProtoNullValue value: " + value); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PType.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPType.java similarity index 79% rename from core/src/main/java/io/github/dfa1/vortex/proto/PType.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPType.java index 6aabc49c4..cd5d639f1 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PType.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPType.java @@ -5,7 +5,7 @@ /// Generated from proto3 enum {@code vortex.dtype.PType}. /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public enum PType { +public enum ProtoPType { U8(0), U16(1), U32(2), @@ -20,7 +20,7 @@ public enum PType { private final int value; - PType(int value) { + ProtoPType(int value) { this.value = value; } @@ -33,12 +33,12 @@ public int value() { /// @param value wire-format integer /// @return matching enum constant /// @throws IllegalArgumentException if no constant matches - public static PType fromValue(int value) { - for (PType v : values()) { + public static ProtoPType fromValue(int value) { + for (ProtoPType v : values()) { if (v.value == value) { return v; } } - throw new IllegalArgumentException("unknown PType value: " + value); + throw new IllegalArgumentException("unknown ProtoPType value: " + value); } } diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PatchedMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchedMetadata.java similarity index 90% rename from core/src/main/java/io/github/dfa1/vortex/proto/PatchedMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchedMetadata.java index 624f83a45..4141e6d83 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PatchedMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchedMetadata.java @@ -10,7 +10,7 @@ /// @param n_lanes field tag 2 /// @param offset field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record PatchedMetadata( +public record ProtoPatchedMetadata( int n_patches, int n_lanes, int offset @@ -22,7 +22,7 @@ public record PatchedMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static PatchedMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPatchedMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int n_patches = 0; int n_lanes = 0; @@ -42,7 +42,7 @@ public static PatchedMetadata decode(MemorySegment __seg, long __off, long __len default -> r.skipField(tag & 7); } } - return new PatchedMetadata(n_patches, n_lanes, offset); + return new ProtoPatchedMetadata(n_patches, n_lanes, offset); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PatchesMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchesMetadata.java similarity index 80% rename from core/src/main/java/io/github/dfa1/vortex/proto/PatchesMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchesMetadata.java index b4fbaed5e..9a8ab9697 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PatchesMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPatchesMetadata.java @@ -13,12 +13,12 @@ /// @param chunk_offsets_ptype field tag 5 /// @param offset_within_chunk field tag 6 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record PatchesMetadata( +public record ProtoPatchesMetadata( long len, long offset, - PType indices_ptype, + ProtoPType indices_ptype, Long chunk_offsets_len, - PType chunk_offsets_ptype, + ProtoPType chunk_offsets_ptype, Long offset_within_chunk ) { @@ -28,13 +28,13 @@ public record PatchesMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static PatchesMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPatchesMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long len = 0; long offset = 0; - PType indices_ptype = PType.U8; + ProtoPType indices_ptype = ProtoPType.U8; Long chunk_offsets_len = null; - PType chunk_offsets_ptype = null; + ProtoPType chunk_offsets_ptype = null; Long offset_within_chunk = null; while (r.hasMore()) { int tag = r.readVarint32(); @@ -48,9 +48,9 @@ public static PatchesMetadata decode(MemorySegment __seg, long __off, long __len case 3 -> { int __ev = r.readVarint32(); try { - indices_ptype = PType.fromValue(__ev); + indices_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 4 -> { @@ -59,9 +59,9 @@ public static PatchesMetadata decode(MemorySegment __seg, long __off, long __len case 5 -> { int __ev = r.readVarint32(); try { - chunk_offsets_ptype = PType.fromValue(__ev); + chunk_offsets_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 6 -> { @@ -70,7 +70,7 @@ public static PatchesMetadata decode(MemorySegment __seg, long __off, long __len default -> r.skipField(tag & 7); } } - return new PatchesMetadata(len, offset, indices_ptype, chunk_offsets_len, chunk_offsets_ptype, offset_within_chunk); + return new ProtoPatchesMetadata(len, offset, indices_ptype, chunk_offsets_len, chunk_offsets_ptype, offset_within_chunk); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PcoChunkInfo.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoChunkInfo.java similarity index 77% rename from core/src/main/java/io/github/dfa1/vortex/proto/PcoChunkInfo.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoChunkInfo.java index 03d277927..6ab1f15cc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PcoChunkInfo.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoChunkInfo.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param pages field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record PcoChunkInfo( - java.util.List pages +public record ProtoPcoChunkInfo( + java.util.List pages ) { /// Decodes a {@code vortex.encodings.PcoChunkInfo} from a slice of a memory segment. @@ -18,20 +18,20 @@ public record PcoChunkInfo( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static PcoChunkInfo decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPcoChunkInfo decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - java.util.List pages = new java.util.ArrayList<>(); + java.util.List pages = new java.util.ArrayList<>(); while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - pages.add(PcoPageInfo.decode(__slice, 0, __slice.byteSize())); + pages.add(ProtoPcoPageInfo.decode(__slice, 0, __slice.byteSize())); } default -> r.skipField(tag & 7); } } - return new PcoChunkInfo(pages); + return new ProtoPcoChunkInfo(pages); } /// Encodes this record to a proto3-wire-format byte array. @@ -43,7 +43,7 @@ public byte[] encode() { } void encodeTo(ProtoWriter w) { - for (PcoPageInfo __v : pages) { + for (ProtoPcoPageInfo __v : pages) { w.writeTag(1, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PcoMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoMetadata.java similarity index 81% rename from core/src/main/java/io/github/dfa1/vortex/proto/PcoMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoMetadata.java index 159073d0b..5e8b3bf06 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PcoMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoMetadata.java @@ -10,9 +10,9 @@ /// @param header field tag 1 /// @param chunks field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record PcoMetadata( +public record ProtoPcoMetadata( byte[] header, - java.util.List chunks + java.util.List chunks ) { /// Decodes a {@code vortex.encodings.PcoMetadata} from a slice of a memory segment. @@ -21,10 +21,10 @@ public record PcoMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static PcoMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPcoMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); byte[] header = new byte[0]; - java.util.List chunks = new java.util.ArrayList<>(); + java.util.List chunks = new java.util.ArrayList<>(); while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -33,12 +33,12 @@ public static PcoMetadata decode(MemorySegment __seg, long __off, long __len) th } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - chunks.add(PcoChunkInfo.decode(__slice, 0, __slice.byteSize())); + chunks.add(ProtoPcoChunkInfo.decode(__slice, 0, __slice.byteSize())); } default -> r.skipField(tag & 7); } } - return new PcoMetadata(header, chunks); + return new ProtoPcoMetadata(header, chunks); } /// Encodes this record to a proto3-wire-format byte array. @@ -54,7 +54,7 @@ void encodeTo(ProtoWriter w) { w.writeTag(1, 2); w.writeBytes(header); } - for (PcoChunkInfo __v : chunks) { + for (ProtoPcoChunkInfo __v : chunks) { w.writeTag(2, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); @@ -67,7 +67,7 @@ public boolean equals(Object __o) { if (this == __o) { return true; } - if (!(__o instanceof PcoMetadata __that)) { + if (!(__o instanceof ProtoPcoMetadata __that)) { return false; } return java.util.Arrays.equals(header, __that.header) diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/PcoPageInfo.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoPageInfo.java similarity index 88% rename from core/src/main/java/io/github/dfa1/vortex/proto/PcoPageInfo.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoPageInfo.java index 3e6456a07..497a69fa6 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/PcoPageInfo.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPcoPageInfo.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param n_values field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record PcoPageInfo( +public record ProtoPcoPageInfo( int n_values ) { @@ -18,7 +18,7 @@ public record PcoPageInfo( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static PcoPageInfo decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPcoPageInfo decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int n_values = 0; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static PcoPageInfo decode(MemorySegment __seg, long __off, long __len) th default -> r.skipField(tag & 7); } } - return new PcoPageInfo(n_values); + return new ProtoPcoPageInfo(n_values); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Primitive.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPrimitive.java similarity index 82% rename from core/src/main/java/io/github/dfa1/vortex/proto/Primitive.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoPrimitive.java index 30c64bc5f..83049542a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Primitive.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoPrimitive.java @@ -9,8 +9,8 @@ /// @param type field tag 1 /// @param nullable field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Primitive( - PType type, +public record ProtoPrimitive( + ProtoPType type, boolean nullable ) { @@ -20,9 +20,9 @@ public record Primitive( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Primitive decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoPrimitive decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType type = PType.U8; + ProtoPType type = ProtoPType.U8; boolean nullable = false; while (r.hasMore()) { int tag = r.readVarint32(); @@ -30,9 +30,9 @@ public static Primitive decode(MemorySegment __seg, long __off, long __len) thro case 1 -> { int __ev = r.readVarint32(); try { - type = PType.fromValue(__ev); + type = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 2 -> { @@ -41,7 +41,7 @@ public static Primitive decode(MemorySegment __seg, long __off, long __len) thro default -> r.skipField(tag & 7); } } - return new Primitive(type, nullable); + return new ProtoPrimitive(type, nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/RLEMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoRLEMetadata.java similarity index 80% rename from core/src/main/java/io/github/dfa1/vortex/proto/RLEMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoRLEMetadata.java index 3a2f69cdb..80b3b1906 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/RLEMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoRLEMetadata.java @@ -13,12 +13,12 @@ /// @param values_idx_offsets_ptype field tag 5 /// @param offset field tag 6 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record RLEMetadata( +public record ProtoRLEMetadata( long values_len, long indices_len, - PType indices_ptype, + ProtoPType indices_ptype, long values_idx_offsets_len, - PType values_idx_offsets_ptype, + ProtoPType values_idx_offsets_ptype, long offset ) { @@ -28,13 +28,13 @@ public record RLEMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static RLEMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoRLEMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long values_len = 0; long indices_len = 0; - PType indices_ptype = PType.U8; + ProtoPType indices_ptype = ProtoPType.U8; long values_idx_offsets_len = 0; - PType values_idx_offsets_ptype = PType.U8; + ProtoPType values_idx_offsets_ptype = ProtoPType.U8; long offset = 0; while (r.hasMore()) { int tag = r.readVarint32(); @@ -48,9 +48,9 @@ public static RLEMetadata decode(MemorySegment __seg, long __off, long __len) th case 3 -> { int __ev = r.readVarint32(); try { - indices_ptype = PType.fromValue(__ev); + indices_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 4 -> { @@ -59,9 +59,9 @@ public static RLEMetadata decode(MemorySegment __seg, long __off, long __len) th case 5 -> { int __ev = r.readVarint32(); try { - values_idx_offsets_ptype = PType.fromValue(__ev); + values_idx_offsets_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 6 -> { @@ -70,7 +70,7 @@ public static RLEMetadata decode(MemorySegment __seg, long __off, long __len) th default -> r.skipField(tag & 7); } } - return new RLEMetadata(values_len, indices_len, indices_ptype, values_idx_offsets_len, values_idx_offsets_ptype, offset); + return new ProtoRLEMetadata(values_len, indices_len, indices_ptype, values_idx_offsets_len, values_idx_offsets_ptype, offset); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/RunEndMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoRunEndMetadata.java similarity index 82% rename from core/src/main/java/io/github/dfa1/vortex/proto/RunEndMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoRunEndMetadata.java index 37a8a81f7..775cc9dd3 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/RunEndMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoRunEndMetadata.java @@ -10,8 +10,8 @@ /// @param num_runs field tag 2 /// @param offset field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record RunEndMetadata( - PType ends_ptype, +public record ProtoRunEndMetadata( + ProtoPType ends_ptype, long num_runs, long offset ) { @@ -22,9 +22,9 @@ public record RunEndMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static RunEndMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoRunEndMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType ends_ptype = PType.U8; + ProtoPType ends_ptype = ProtoPType.U8; long num_runs = 0; long offset = 0; while (r.hasMore()) { @@ -33,9 +33,9 @@ public static RunEndMetadata decode(MemorySegment __seg, long __off, long __len) case 1 -> { int __ev = r.readVarint32(); try { - ends_ptype = PType.fromValue(__ev); + ends_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } case 2 -> { @@ -47,7 +47,7 @@ public static RunEndMetadata decode(MemorySegment __seg, long __off, long __len) default -> r.skipField(tag & 7); } } - return new RunEndMetadata(ends_ptype, num_runs, offset); + return new ProtoRunEndMetadata(ends_ptype, num_runs, offset); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Scalar.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalar.java similarity index 79% rename from core/src/main/java/io/github/dfa1/vortex/proto/Scalar.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalar.java index 2d5fac8aa..d2aa6aae8 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Scalar.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalar.java @@ -9,9 +9,9 @@ /// @param dtype field tag 1 /// @param value field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Scalar( - DType dtype, - ScalarValue value +public record ProtoScalar( + ProtoDType dtype, + ProtoScalarValue value ) { /// Decodes a {@code vortex.scalar.Scalar} from a slice of a memory segment. @@ -20,25 +20,25 @@ public record Scalar( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Scalar decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoScalar decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - DType dtype = null; - ScalarValue value = null; + ProtoDType dtype = null; + ProtoScalarValue value = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - dtype = DType.decode(__slice, 0, __slice.byteSize()); + dtype = ProtoDType.decode(__slice, 0, __slice.byteSize()); } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - value = ScalarValue.decode(__slice, 0, __slice.byteSize()); + value = ProtoScalarValue.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new Scalar(dtype, value); + return new ProtoScalar(dtype, value); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ScalarValue.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalarValue.java similarity index 75% rename from core/src/main/java/io/github/dfa1/vortex/proto/ScalarValue.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalarValue.java index cee8fcfa9..4b51b6227 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ScalarValue.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoScalarValue.java @@ -19,8 +19,8 @@ /// @param f16_value field tag 10 /// @param variant_value field tag 11 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ScalarValue( - NullValue null_value, +public record ProtoScalarValue( + ProtoNullValue null_value, Boolean bool_value, Long int64_value, Long uint64_value, @@ -28,9 +28,9 @@ public record ScalarValue( Double f64_value, String string_value, byte[] bytes_value, - ListValue list_value, + ProtoListValue list_value, Long f16_value, - Scalar variant_value + ProtoScalar variant_value ) { /// Decodes a {@code vortex.scalar.ScalarValue} from a slice of a memory segment. @@ -39,9 +39,9 @@ public record ScalarValue( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ScalarValue decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoScalarValue decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - NullValue null_value = null; + ProtoNullValue null_value = null; Boolean bool_value = null; Long int64_value = null; Long uint64_value = null; @@ -49,18 +49,18 @@ public static ScalarValue decode(MemorySegment __seg, long __off, long __len) th Double f64_value = null; String string_value = null; byte[] bytes_value = null; - ListValue list_value = null; + ProtoListValue list_value = null; Long f16_value = null; - Scalar variant_value = null; + ProtoScalar variant_value = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { int __ev = r.readVarint32(); try { - null_value = NullValue.fromValue(__ev); + null_value = ProtoNullValue.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown NullValue value: " + __ev); + throw new IOException("unknown ProtoNullValue value: " + __ev); } } case 2 -> { @@ -86,19 +86,19 @@ public static ScalarValue decode(MemorySegment __seg, long __off, long __len) th } case 9 -> { MemorySegment __slice = r.readLenDelimSegment(); - list_value = ListValue.decode(__slice, 0, __slice.byteSize()); + list_value = ProtoListValue.decode(__slice, 0, __slice.byteSize()); } case 10 -> { f16_value = r.readVarint64(); } case 11 -> { MemorySegment __slice = r.readLenDelimSegment(); - variant_value = Scalar.decode(__slice, 0, __slice.byteSize()); + variant_value = ProtoScalar.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new ScalarValue(null_value, bool_value, int64_value, uint64_value, f32_value, f64_value, string_value, bytes_value, list_value, f16_value, variant_value); + return new ProtoScalarValue(null_value, bool_value, int64_value, uint64_value, f32_value, f64_value, string_value, bytes_value, list_value, f16_value, variant_value); } /// Encodes this record to a proto3-wire-format byte array. @@ -163,78 +163,78 @@ void encodeTo(ProtoWriter w) { /// Factory for oneof case {@code null_value} (field tag 1). /// @param value the value to set /// @return a record with only the {@code null_value} component set - public static ScalarValue ofNullValue(NullValue value) { - return new ScalarValue(value, null, null, null, null, null, null, null, null, null, null); + public static ProtoScalarValue ofNullValue(ProtoNullValue value) { + return new ProtoScalarValue(value, null, null, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code bool_value} (field tag 2). /// @param value the value to set /// @return a record with only the {@code bool_value} component set - public static ScalarValue ofBoolValue(Boolean value) { - return new ScalarValue(null, value, null, null, null, null, null, null, null, null, null); + public static ProtoScalarValue ofBoolValue(Boolean value) { + return new ProtoScalarValue(null, value, null, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code int64_value} (field tag 3). /// @param value the value to set /// @return a record with only the {@code int64_value} component set - public static ScalarValue ofInt64Value(Long value) { - return new ScalarValue(null, null, value, null, null, null, null, null, null, null, null); + public static ProtoScalarValue ofInt64Value(Long value) { + return new ProtoScalarValue(null, null, value, null, null, null, null, null, null, null, null); } /// Factory for oneof case {@code uint64_value} (field tag 4). /// @param value the value to set /// @return a record with only the {@code uint64_value} component set - public static ScalarValue ofUint64Value(Long value) { - return new ScalarValue(null, null, null, value, null, null, null, null, null, null, null); + public static ProtoScalarValue ofUint64Value(Long value) { + return new ProtoScalarValue(null, null, null, value, null, null, null, null, null, null, null); } /// Factory for oneof case {@code f32_value} (field tag 5). /// @param value the value to set /// @return a record with only the {@code f32_value} component set - public static ScalarValue ofF32Value(Float value) { - return new ScalarValue(null, null, null, null, value, null, null, null, null, null, null); + public static ProtoScalarValue ofF32Value(Float value) { + return new ProtoScalarValue(null, null, null, null, value, null, null, null, null, null, null); } /// Factory for oneof case {@code f64_value} (field tag 6). /// @param value the value to set /// @return a record with only the {@code f64_value} component set - public static ScalarValue ofF64Value(Double value) { - return new ScalarValue(null, null, null, null, null, value, null, null, null, null, null); + public static ProtoScalarValue ofF64Value(Double value) { + return new ProtoScalarValue(null, null, null, null, null, value, null, null, null, null, null); } /// Factory for oneof case {@code string_value} (field tag 7). /// @param value the value to set /// @return a record with only the {@code string_value} component set - public static ScalarValue ofStringValue(String value) { - return new ScalarValue(null, null, null, null, null, null, value, null, null, null, null); + public static ProtoScalarValue ofStringValue(String value) { + return new ProtoScalarValue(null, null, null, null, null, null, value, null, null, null, null); } /// Factory for oneof case {@code bytes_value} (field tag 8). /// @param value the value to set /// @return a record with only the {@code bytes_value} component set - public static ScalarValue ofBytesValue(byte[] value) { - return new ScalarValue(null, null, null, null, null, null, null, value, null, null, null); + public static ProtoScalarValue ofBytesValue(byte[] value) { + return new ProtoScalarValue(null, null, null, null, null, null, null, value, null, null, null); } /// Factory for oneof case {@code list_value} (field tag 9). /// @param value the value to set /// @return a record with only the {@code list_value} component set - public static ScalarValue ofListValue(ListValue value) { - return new ScalarValue(null, null, null, null, null, null, null, null, value, null, null); + public static ProtoScalarValue ofListValue(ProtoListValue value) { + return new ProtoScalarValue(null, null, null, null, null, null, null, null, value, null, null); } /// Factory for oneof case {@code f16_value} (field tag 10). /// @param value the value to set /// @return a record with only the {@code f16_value} component set - public static ScalarValue ofF16Value(Long value) { - return new ScalarValue(null, null, null, null, null, null, null, null, null, value, null); + public static ProtoScalarValue ofF16Value(Long value) { + return new ProtoScalarValue(null, null, null, null, null, null, null, null, null, value, null); } /// Factory for oneof case {@code variant_value} (field tag 11). /// @param value the value to set /// @return a record with only the {@code variant_value} component set - public static ScalarValue ofVariantValue(Scalar value) { - return new ScalarValue(null, null, null, null, null, null, null, null, null, null, value); + public static ProtoScalarValue ofVariantValue(ProtoScalar value) { + return new ProtoScalarValue(null, null, null, null, null, null, null, null, null, null, value); } @Override @@ -242,7 +242,7 @@ public boolean equals(Object __o) { if (this == __o) { return true; } - if (!(__o instanceof ScalarValue __that)) { + if (!(__o instanceof ProtoScalarValue __that)) { return false; } return java.util.Objects.equals(null_value, __that.null_value) diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/SequenceMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoSequenceMetadata.java similarity index 77% rename from core/src/main/java/io/github/dfa1/vortex/proto/SequenceMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoSequenceMetadata.java index 26cfd5802..fd21c4308 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/SequenceMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoSequenceMetadata.java @@ -9,9 +9,9 @@ /// @param base field tag 1 /// @param multiplier field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record SequenceMetadata( - ScalarValue base, - ScalarValue multiplier +public record ProtoSequenceMetadata( + ProtoScalarValue base, + ProtoScalarValue multiplier ) { /// Decodes a {@code vortex.encodings.SequenceMetadata} from a slice of a memory segment. @@ -20,25 +20,25 @@ public record SequenceMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static SequenceMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoSequenceMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - ScalarValue base = null; - ScalarValue multiplier = null; + ProtoScalarValue base = null; + ProtoScalarValue multiplier = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - base = ScalarValue.decode(__slice, 0, __slice.byteSize()); + base = ProtoScalarValue.decode(__slice, 0, __slice.byteSize()); } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - multiplier = ScalarValue.decode(__slice, 0, __slice.byteSize()); + multiplier = ProtoScalarValue.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new SequenceMetadata(base, multiplier); + return new ProtoSequenceMetadata(base, multiplier); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/SparseMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoSparseMetadata.java similarity index 80% rename from core/src/main/java/io/github/dfa1/vortex/proto/SparseMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoSparseMetadata.java index df1e6e2a9..6db30babe 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/SparseMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoSparseMetadata.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param patches field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record SparseMetadata( - PatchesMetadata patches +public record ProtoSparseMetadata( + ProtoPatchesMetadata patches ) { /// Decodes a {@code vortex.encodings.SparseMetadata} from a slice of a memory segment. @@ -18,20 +18,20 @@ public record SparseMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static SparseMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoSparseMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PatchesMetadata patches = null; + ProtoPatchesMetadata patches = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - patches = PatchesMetadata.decode(__slice, 0, __slice.byteSize()); + patches = ProtoPatchesMetadata.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new SparseMetadata(patches); + return new ProtoSparseMetadata(patches); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Struct.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoStruct.java similarity index 83% rename from core/src/main/java/io/github/dfa1/vortex/proto/Struct.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoStruct.java index 608c96f45..f8b5deae5 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Struct.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoStruct.java @@ -10,9 +10,9 @@ /// @param dtypes field tag 2 /// @param nullable field tag 3 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Struct( +public record ProtoStruct( java.util.List names, - java.util.List dtypes, + java.util.List dtypes, boolean nullable ) { @@ -22,10 +22,10 @@ public record Struct( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Struct decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoStruct decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); java.util.List names = new java.util.ArrayList<>(); - java.util.List dtypes = new java.util.ArrayList<>(); + java.util.List dtypes = new java.util.ArrayList<>(); boolean nullable = false; while (r.hasMore()) { int tag = r.readVarint32(); @@ -35,7 +35,7 @@ public static Struct decode(MemorySegment __seg, long __off, long __len) throws } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - dtypes.add(DType.decode(__slice, 0, __slice.byteSize())); + dtypes.add(ProtoDType.decode(__slice, 0, __slice.byteSize())); } case 3 -> { nullable = r.readBool(); @@ -43,7 +43,7 @@ public static Struct decode(MemorySegment __seg, long __off, long __len) throws default -> r.skipField(tag & 7); } } - return new Struct(names, dtypes, nullable); + return new ProtoStruct(names, dtypes, nullable); } /// Encodes this record to a proto3-wire-format byte array. @@ -59,7 +59,7 @@ void encodeTo(ProtoWriter w) { w.writeTag(1, 2); w.writeString(__v); } - for (DType __v : dtypes) { + for (ProtoDType __v : dtypes) { w.writeTag(2, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Union.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoUnion.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/Union.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoUnion.java index 58873ccb6..736fe491d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Union.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoUnion.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param nullable field tag 4 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Union( +public record ProtoUnion( boolean nullable ) { @@ -18,7 +18,7 @@ public record Union( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Union decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoUnion decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); boolean nullable = false; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Union decode(MemorySegment __seg, long __off, long __len) throws I default -> r.skipField(tag & 7); } } - return new Union(nullable); + return new ProtoUnion(nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Utf8.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoUtf8.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/Utf8.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoUtf8.java index c3b363333..2ce34c0e8 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Utf8.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoUtf8.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param nullable field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Utf8( +public record ProtoUtf8( boolean nullable ) { @@ -18,7 +18,7 @@ public record Utf8( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Utf8 decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoUtf8 decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); boolean nullable = false; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Utf8 decode(MemorySegment __seg, long __off, long __len) throws IO default -> r.skipField(tag & 7); } } - return new Utf8(nullable); + return new ProtoUtf8(nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/VarBinMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVarBinMetadata.java similarity index 78% rename from core/src/main/java/io/github/dfa1/vortex/proto/VarBinMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoVarBinMetadata.java index 5805174ec..ce5c7dce2 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/VarBinMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVarBinMetadata.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param offsets_ptype field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record VarBinMetadata( - PType offsets_ptype +public record ProtoVarBinMetadata( + ProtoPType offsets_ptype ) { /// Decodes a {@code vortex.encodings.VarBinMetadata} from a slice of a memory segment. @@ -18,24 +18,24 @@ public record VarBinMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static VarBinMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoVarBinMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - PType offsets_ptype = PType.U8; + ProtoPType offsets_ptype = ProtoPType.U8; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { int __ev = r.readVarint32(); try { - offsets_ptype = PType.fromValue(__ev); + offsets_ptype = ProtoPType.fromValue(__ev); } catch (IllegalArgumentException __iae) { - throw new IOException("unknown PType value: " + __ev); + throw new IOException("unknown ProtoPType value: " + __ev); } } default -> r.skipField(tag & 7); } } - return new VarBinMetadata(offsets_ptype); + return new ProtoVarBinMetadata(offsets_ptype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/Variant.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariant.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/Variant.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariant.java index b5a23cb86..11f235208 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/Variant.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariant.java @@ -8,7 +8,7 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param nullable field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record Variant( +public record ProtoVariant( boolean nullable ) { @@ -18,7 +18,7 @@ public record Variant( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static Variant decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoVariant decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); boolean nullable = false; while (r.hasMore()) { @@ -30,7 +30,7 @@ public static Variant decode(MemorySegment __seg, long __off, long __len) throws default -> r.skipField(tag & 7); } } - return new Variant(nullable); + return new ProtoVariant(nullable); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/VariantMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariantMetadata.java similarity index 81% rename from core/src/main/java/io/github/dfa1/vortex/proto/VariantMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariantMetadata.java index fb262749a..7ae7c9407 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/VariantMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoVariantMetadata.java @@ -8,8 +8,8 @@ /// Do not edit by hand — regenerate via {@code ./mvnw generate-sources -pl core -P regenerate-sources}. /// @param shredded_dtype field tag 1 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record VariantMetadata( - DType shredded_dtype +public record ProtoVariantMetadata( + ProtoDType shredded_dtype ) { /// Decodes a {@code vortex.encodings.VariantMetadata} from a slice of a memory segment. @@ -18,20 +18,20 @@ public record VariantMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static VariantMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoVariantMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); - DType shredded_dtype = null; + ProtoDType shredded_dtype = null; while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { case 1 -> { MemorySegment __slice = r.readLenDelimSegment(); - shredded_dtype = DType.decode(__slice, 0, __slice.byteSize()); + shredded_dtype = ProtoDType.decode(__slice, 0, __slice.byteSize()); } default -> r.skipField(tag & 7); } } - return new VariantMetadata(shredded_dtype); + return new ProtoVariantMetadata(shredded_dtype); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ZstdFrameMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdFrameMetadata.java similarity index 89% rename from core/src/main/java/io/github/dfa1/vortex/proto/ZstdFrameMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdFrameMetadata.java index 63c1bcb06..9ed04ba7b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ZstdFrameMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdFrameMetadata.java @@ -9,7 +9,7 @@ /// @param uncompressed_size field tag 1 /// @param n_values field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ZstdFrameMetadata( +public record ProtoZstdFrameMetadata( long uncompressed_size, long n_values ) { @@ -20,7 +20,7 @@ public record ZstdFrameMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ZstdFrameMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoZstdFrameMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); long uncompressed_size = 0; long n_values = 0; @@ -36,7 +36,7 @@ public static ZstdFrameMetadata decode(MemorySegment __seg, long __off, long __l default -> r.skipField(tag & 7); } } - return new ZstdFrameMetadata(uncompressed_size, n_values); + return new ProtoZstdFrameMetadata(uncompressed_size, n_values); } /// Encodes this record to a proto3-wire-format byte array. diff --git a/core/src/main/java/io/github/dfa1/vortex/proto/ZstdMetadata.java b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdMetadata.java similarity index 78% rename from core/src/main/java/io/github/dfa1/vortex/proto/ZstdMetadata.java rename to core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdMetadata.java index db0b56017..dd31325a1 100644 --- a/core/src/main/java/io/github/dfa1/vortex/proto/ZstdMetadata.java +++ b/core/src/main/java/io/github/dfa1/vortex/proto/ProtoZstdMetadata.java @@ -9,9 +9,9 @@ /// @param dictionary_size field tag 1 /// @param frames field tag 2 @Generated("io.github.dfa1.vortex.protogen.CodeGen") -public record ZstdMetadata( +public record ProtoZstdMetadata( int dictionary_size, - java.util.List frames + java.util.List frames ) { /// Decodes a {@code vortex.encodings.ZstdMetadata} from a slice of a memory segment. @@ -20,10 +20,10 @@ public record ZstdMetadata( /// @param __len payload length in bytes /// @return decoded record /// @throws IOException if the slice is malformed or truncated - public static ZstdMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { + public static ProtoZstdMetadata decode(MemorySegment __seg, long __off, long __len) throws IOException { ProtoReader r = new ProtoReader(__seg, __off, __len); int dictionary_size = 0; - java.util.List frames = new java.util.ArrayList<>(); + java.util.List frames = new java.util.ArrayList<>(); while (r.hasMore()) { int tag = r.readVarint32(); switch (tag >>> 3) { @@ -32,12 +32,12 @@ public static ZstdMetadata decode(MemorySegment __seg, long __off, long __len) t } case 2 -> { MemorySegment __slice = r.readLenDelimSegment(); - frames.add(ZstdFrameMetadata.decode(__slice, 0, __slice.byteSize())); + frames.add(ProtoZstdFrameMetadata.decode(__slice, 0, __slice.byteSize())); } default -> r.skipField(tag & 7); } } - return new ZstdMetadata(dictionary_size, frames); + return new ProtoZstdMetadata(dictionary_size, frames); } /// Encodes this record to a proto3-wire-format byte array. @@ -53,7 +53,7 @@ void encodeTo(ProtoWriter w) { w.writeTag(1, 0); w.writeVarint32(dictionary_size); } - for (ZstdFrameMetadata __v : frames) { + for (ProtoZstdFrameMetadata __v : frames) { w.writeTag(2, 2); int __mark = w.beginLenDelim(); __v.encodeTo(w); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/DTypeExtensionTest.java b/core/src/test/java/io/github/dfa1/vortex/core/DTypeExtensionTest.java index 1d701ef1e..b0c67519e 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/DTypeExtensionTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/DTypeExtensionTest.java @@ -2,7 +2,7 @@ import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -15,10 +15,10 @@ class DTypeExtensionTest { void metadataAtCap_accepted() { // Given — exact boundary must pass so legitimate large-but-bounded metadata // is not falsely rejected - ByteBuffer meta = ByteBuffer.allocate(DType.Extension.MAX_METADATA_SIZE); + MemorySegment meta = MemorySegment.ofArray(new byte[DType.Extension.MAX_METADATA_SIZE]); // When / Then - assertThat(new DType.Extension("acme.big", I32, meta, false).metadata().remaining()) + assertThat(new DType.Extension("acme.big", I32, meta, false).metadata().byteSize()) .isEqualTo(DType.Extension.MAX_METADATA_SIZE); } @@ -26,7 +26,7 @@ void metadataAtCap_accepted() { void metadataOverCap_rejected() { // Given — one byte past the cap; defends parser against attacker-supplied // multi-megabyte metadata blobs that would inflate parse-time allocations - ByteBuffer meta = ByteBuffer.allocate(DType.Extension.MAX_METADATA_SIZE + 1); + MemorySegment meta = MemorySegment.ofArray(new byte[DType.Extension.MAX_METADATA_SIZE + 1]); // When / Then assertThatThrownBy(() -> new DType.Extension("acme.big", I32, meta, false)) diff --git a/core/src/test/java/io/github/dfa1/vortex/proto/ProtoRuntimeTest.java b/core/src/test/java/io/github/dfa1/vortex/proto/ProtoRuntimeTest.java index 4f329a5cd..a15262642 100644 --- a/core/src/test/java/io/github/dfa1/vortex/proto/ProtoRuntimeTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/proto/ProtoRuntimeTest.java @@ -272,16 +272,16 @@ class UnknownEnum { @Test void unknownPTypeValueIsCheckedIOException() { - // Given — Primitive { type = 99 } where PType has no constant for 99. - // Field tag 1, wire type VARINT (0): tag byte = (1 << 3) | 0 = 0x08, value 99 = 0x63. + // Given — ProtoPrimitive { type = 99 } where ProtoPType has no constant for 99. + // ProtoField tag 1, wire type VARINT (0): tag byte = (1 << 3) | 0 = 0x08, value 99 = 0x63. byte[] wire = new byte[]{0x08, 0x63}; MemorySegment seg = MemorySegment.ofArray(wire); // When + Then — must be checked IOException (per SECURITY.md guarantee), // not the underlying IllegalArgumentException from Enum.fromValue. - assertThatThrownBy(() -> Primitive.decode(seg, 0, wire.length)) + assertThatThrownBy(() -> ProtoPrimitive.decode(seg, 0, wire.length)) .isInstanceOf(IOException.class) - .hasMessageContaining("PType"); + .hasMessageContaining("ProtoPType"); } } @@ -290,10 +290,10 @@ class SingleNullEncode { @Test void singleStringNullEncodesEmpty() { - // Given — Extension with all SINGLE fields null. Pre-fix this NPE'd on + // Given — ProtoExtension with all SINGLE fields null. Pre-fix this NPE'd on // String.isEmpty() / byte[].length. Default-value SINGLE fields must skip // emitting the tag entirely. - Extension ext = new Extension(null, null, null); + ProtoExtension ext = new ProtoExtension(null, null, null); // When byte[] wire = ext.encode(); @@ -311,8 +311,8 @@ void recordsWithEqualByteArraysAreEqual() { // Given — records auto-equals would do reference compare on byte[]. The // generator overrides equals/hashCode with Arrays.equals/Arrays.hashCode // so structurally equal records compare equal. - ScalarValue a = ScalarValue.ofBytesValue(new byte[]{1, 2, 3}); - ScalarValue b = ScalarValue.ofBytesValue(new byte[]{1, 2, 3}); + ProtoScalarValue a = ProtoScalarValue.ofBytesValue(new byte[]{1, 2, 3}); + ProtoScalarValue b = ProtoScalarValue.ofBytesValue(new byte[]{1, 2, 3}); // When + Then assertThat(a).isEqualTo(b); @@ -322,8 +322,8 @@ void recordsWithEqualByteArraysAreEqual() { @Test void recordsWithDifferentByteArraysAreNotEqual() { // Given - ScalarValue a = ScalarValue.ofBytesValue(new byte[]{1, 2, 3}); - ScalarValue b = ScalarValue.ofBytesValue(new byte[]{1, 2, 4}); + ProtoScalarValue a = ProtoScalarValue.ofBytesValue(new byte[]{1, 2, 3}); + ProtoScalarValue b = ProtoScalarValue.ofBytesValue(new byte[]{1, 2, 4}); // When + Then assertThat(a).isNotEqualTo(b); diff --git a/docs/adr/0017-in-house-flatbuffers-codegen.md b/docs/adr/0017-in-house-flatbuffers-codegen.md new file mode 100644 index 000000000..5f5e78639 --- /dev/null +++ b/docs/adr/0017-in-house-flatbuffers-codegen.md @@ -0,0 +1,169 @@ +# ADR 0017: In-house FlatBuffers codegen + MemorySegment-native runtime + +- **Status:** Accepted +- **Date:** 2026-06-23 +- **Deciders:** project maintainer +- **Supersedes:** — +- **Superseded by:** — +- **Related:** ADR 0001 (split read/write runtimes), ADR 0011 (writer zero-copy + MemorySegment overload). Mirrors the existing in-house `proto-gen`. + +## Context + +The file format is described by two schema languages: FlatBuffers (`.fbs` — +Footer, Layout, Postscript, Array) and Protobuf (`.proto` — DType, ScalarValue, +encoding metadata). Protobuf is already self-hosted: `proto-gen` parses `.proto` +and emits records with `decode(MemorySegment, long, long)` / `encode()` that +operate directly on a `MemorySegment`, with no `protobuf-java` runtime. + +FlatBuffers is not. We still depend on: + +- **`flatc`** (the schema compiler) at code-gen time — an external native tool + installed via `brew install flatbuffers`, gated behind the + `regenerate-sources` profile, with a version-guard strip hack in `core/pom.xml` + because `flatbuffers-java` on Maven Central lags the `flatc` CLI by months. +- **`com.google.flatbuffers:flatbuffers-java`** as a runtime dependency. + +Two forces make this dependency costly out of proportion to the ~32 generated +types we actually use: + +1. **It is not MemorySegment-native.** Generated tables extend + `com.google.flatbuffers.Table` and read through a `java.nio.ByteBuffer`; the + builder writes into a `ByteBuffer`. The reader memory-maps the file into one + `MemorySegment` (ADR 0001 memory model) and the writer is moving toward + MemorySegment output (ADR 0011), so every FlatBuffers access crosses a + `ByteBuffer`↔`MemorySegment` boundary — an extra representation on read and a + bridge+copy on write. The rest of the codebase is FFM-only by policy + (CLAUDE.md: "Uses FFM — never JNI or `sun.misc.Unsafe`"). + +2. **`flatbuffers-java` ships as an automatic module.** It has no + `module-info`; its module name is derived from the jar filename and is + unstable. A proper named JPMS module cannot reliably `requires` an automatic + module and be published to Maven Central. As long as we depend on it, adding + `module-info.java` to hide implementation packages (e.g. the generated + `fbs`/`proto` wire-format classes) is off the table. A patch to give + `flatbuffers-java` a real module descriptor has been submitted upstream but is + **not merged**, so the blocker stands. (See the prefix workaround in the + `Fbs`/`Proto` class-name decision and the "No JPMS" rationale.) + +`proto-gen` already proved the pattern works for a fixed, small schema set we +fully control: a hand-rolled parser + MemorySegment-native emitter is far less +code than a general compiler, and the Rust interop suite is byte-exact ground +truth that makes a self-hosted wire-format implementation safe to attempt. + +## Decision + +Replace `flatc` and `com.google.flatbuffers` with an in-house FlatBuffers +toolchain, mirroring `proto-gen`: + +- **`fbs-gen` module** — a build-time `.fbs` lexer/parser/AST + code generator + (`Lexer`, `Token`, `Ast`, `Parser`, `TypeRegistry`, `CodeGen`, `Main`), not + published. Covers only the schema subset vortex uses. +- **MemorySegment-native runtime in `core`** — `FbsTable` / `FbsStruct` provide + the read primitives (vtable lookup, indirect uoffsets, vectors, unions, inline + structs, UTF-8 strings, zero-copy byte slices) over a `MemorySegment`; an + in-house builder provides the write path (vtable dedup, back-to-front layout, + alignment) emitting straight into a `MemorySegment`. +- **Generated classes** extend that runtime and are emitted with the `Fbs` + name prefix applied **in the generator**, so the `.fbs` schemas revert to + byte-identical with upstream (matching how `proto-gen` keeps `.proto` + pristine). This removes the schema divergence introduced when the prefix was + applied by hand-editing the `.fbs` files. + +The wire format does not change: FlatBuffers binary layout is fixed, and the +emitted bytes must be identical so the Rust reference reads them. + +### Rollout (spike order) + +1. **Parser** — `.fbs` lexer + parser, validated against all four real schemas. +2. **Read runtime** — `FbsTable`/`FbsStruct`, validated against the flatc reader + on bytes produced by the flatc writer (byte-exact cross-check). +3. **CodeGen (read side)** — emit the reader accessors from the AST. +4. **Write builder** — in-house `FlatBufferBuilder` over `MemorySegment`. +5. **Cutover** — switch `core` to the generated MemorySegment classes, drop + `com.google.flatbuffers`, remove `flatc` (and the version-guard strip) from + the `regenerate-sources` profile. + +Each step keeps `main` green; the flatc-generated classes stay in place until +the in-house path matches byte-for-byte on the full integration suite. + +## Consequences + +### Positive + +- **All-FFM.** Both read and write operate on `MemorySegment`; the + `ByteBuffer` bridge in the writer and the `ByteBuffer` view on read both + disappear. Consistent with ADR 0001/0011 and the FFM-only policy. +- **Two external dependencies removed:** the `flatc` native tool (no more + `brew install flatbuffers`, no version-guard strip hack) and the + `flatbuffers-java` runtime jar. +- **JPMS unblocked.** With both wire formats self-hosted, no upstream automatic + module remains; `module-info.java` becomes viable, which in turn lets the + generated `fbs`/`proto` packages be encapsulated at the root instead of merely + prefixed. +- **One codegen pattern.** `fbs-gen` and `proto-gen` share structure; the + prefix lives in the generator for both. +- **Pristine schemas.** `.fbs` no longer diverges from upstream. + +### Negative + +- **A second hand-rolled wire-format implementation to maintain.** This runs + counter to the "`proto-gen` is interim" stance — but the driver here (kill the + automatic module to unblock JPMS, go all-FFM) is stronger than `proto-gen`'s + original JDK-25 motivation. +- **The write builder is non-trivial.** FlatBuffers serializes back-to-front + with vtable dedup and alignment; correctness is all-or-nothing. + +### Risks to manage + +- **Byte-exact correctness.** Mitigation: the Rust interop suite + (`JavaWritesRustReadsIntegrationTest`, `RustJavaReaderComparisonIntegrationTest`, + 275 tests) is the gate; do not cut over until green. Validate read and write + independently against the flatc path before removing it. +- **Schema features we don't yet emit.** The parser/codegen cover only the used + subset; an added schema feature (nested vectors, fixed arrays, key/sorting + attributes) needs a generator change. Acceptable for a fixed in-repo schema + set; fail loudly on unsupported constructs. +- **Upstream patch lands first.** If `flatbuffers-java` gets a real module + descriptor and a lock-step Maven Central release before cutover, the + automatic-module half of the motivation evaporates — but the MemorySegment-native + half still justifies the work. + +## Known limitations + +- **No intra-FlatBuffer bounds checks** in the `fbsrt` read path (e.g. + `FbsTable.readStringAt` allocates `new byte[len]` with `len` from the blob). + Same posture as the previous `com.google.flatbuffers` runtime; file-level framing + is range-checked upstream by `IoBounds`, intra-FlatBuffer fields are not. Trusted-blob + assumption. +- **Inline struct *table* fields are not exercised.** `createX` adds fields by + descending alignment; `FbsBuilder.addStruct` requires the struct be the most + recently written object. Vortex structs (Buffer, SegmentSpec) only appear as vector + elements today, so this never fires. A future inline struct field sharing an + alignment group with a scalar could trip the guard — fix by emitting struct fields + last within their group. Documented at the sort site in `CodeGen`. +- **`FbsBuilder` grows on the heap** (`MemorySegment.ofArray(new byte[])`). Fine: it is + write scaffolding, not the decode hot path the CLAUDE allocation rule targets. +- **Generator unit coverage is light** relative to the surface; the real safety net is + the 275-test Rust-interop suite plus idempotent regeneration and the byte-identical-vs-flatc + builder check in `CodeGenTest`. + +## Alternatives considered + +- **Wait for the upstream `flatbuffers-java` module patch.** Unblocks JPMS + eventually but leaves the `ByteBuffer` boundary and the `flatc`/version-guard + toolchain in place, and has no committed timeline. +- **Shade/relocate `flatbuffers-java` into a named module.** Removes the + automatic-module problem but keeps the `ByteBuffer` runtime and adds a shading + step; does nothing for the FFM goal. +- **Keep `flatc`, write only a MemorySegment runtime adapter.** Still requires + the `flatbuffers-java` types (or a parallel runtime) and the external compiler; + half-measure. + +## References + +- `proto-gen` module (the precedent this mirrors). +- `core/pom.xml` `regenerate-sources` profile — the `flatc` invocation and the + `ValidateVersion()` strip hack this ADR removes. +- FlatBuffers binary format: `google/flatbuffers` `java/.../Table.java`, + `Struct.java`, `FlatBufferBuilder.java`. diff --git a/docs/adr/ADR.md b/docs/adr/ADR.md index 8bfa7b414..e75686a22 100644 --- a/docs/adr/ADR.md +++ b/docs/adr/ADR.md @@ -29,3 +29,4 @@ Each ADR is a Markdown file named `NNNN-short-title.md`. Use `template.md` as th | 0014 | Variant encoding: chunked constants now, parquet.variant later | Completed | 0.8.0 | | 0015 | Drop Materialized fallbacks once Lazy has shipped | Completed | 0.8.0 | | 0016 | vortex-arrow bridge module for Arrow interop | Proposed | | +| 0017 | In-house FlatBuffers codegen + MemorySegment runtime | Accepted | | diff --git a/fbs-gen/pom.xml b/fbs-gen/pom.xml new file mode 100644 index 000000000..894be6b35 --- /dev/null +++ b/fbs-gen/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.9.0-SNAPSHOT + + + vortex-fbs-gen + + vortex-fbs-gen + Build-time .fbs (FlatBuffers schema) to MemorySegment-native Java code generator. Not published. + + + true + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Ast.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Ast.java new file mode 100644 index 000000000..fa2463ccd --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Ast.java @@ -0,0 +1,115 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +/// Container for the `.fbs` AST node records. +/// Nested types are kept here so consumers import a single class. +public final class Ast { + + private Ast() { + } + + /// A parsed `.fbs` schema file. + /// + /// @param namespace value of the `namespace` declaration (dotted), or empty + /// @param includes list of `include "..."` file paths (relative to include root) + /// @param decls declared tables, structs, enums and unions, in source order + /// @param rootTypes names listed in `root_type` declarations + public record SchemaFile(String namespace, List includes, List decls, List rootTypes) { + } + + /// A top-level declaration in a `.fbs` file. + public sealed interface Decl permits TableDecl, StructDecl, EnumDecl, UnionDecl { + + /// @return the declared simple name + String name(); + } + + /// A `table Foo { ... `} — heap-allocated, vtable-indexed, fields optional. + /// + /// @param name simple name + /// @param fields field declarations in source order + public record TableDecl(String name, List fields) implements Decl { + } + + /// A `struct Foo { ... `} — fixed-size, inline, no vtable; every field is present. + /// + /// @param name simple name + /// @param fields field declarations in source order + public record StructDecl(String name, List fields) implements Decl { + } + + /// An `enum Foo: underlying { ... `} declaration. + /// + /// @param name simple name + /// @param underlying the underlying integer scalar (e.g. `uint8`) + /// @param values enum values in source order + public record EnumDecl(String name, Scalar underlying, List values) implements Decl { + } + + /// A single enum value. + /// + /// @param name constant name + /// @param number explicit wire value, or empty for auto-increment + public record EnumVal(String name, OptionalInt number) { + } + + /// A `union Foo { ... `} declaration. On the wire a union is a discriminator byte + /// (`0` = NONE) plus an offset to the selected member table. + /// + /// @param name simple name + /// @param members member types in source order + public record UnionDecl(String name, List members) implements Decl { + } + + /// A single union member. + /// + /// @param typeName referenced table type name + /// @param number explicit discriminator value, or empty for auto-increment (from 1) + public record UnionMember(String typeName, OptionalInt number) { + } + + /// A field declaration inside a table or struct. + /// + /// @param name field name as written + /// @param type field type + /// @param defaultValue literal default as written (`null`, `0`, `true`, ...), or empty + /// @param required whether the field carries the `(required)` attribute + public record Field(String name, FieldType type, Optional defaultValue, boolean required) { + } + + /// Field type variants — a scalar, a vector, or a reference to a declared type. + public sealed interface FieldType permits Scalar, Vector, Ref { + } + + /// FlatBuffers built-in scalar types. Schema aliases (`byte`, `ubyte`, `int`, ...) + /// are normalized to these canonical, width-explicit constants by [Parser]. + public enum Scalar implements FieldType { + BOOL, + INT8, + UINT8, + INT16, + UINT16, + INT32, + UINT32, + INT64, + UINT64, + FLOAT32, + FLOAT64, + STRING + } + + /// A `[element]` vector type. + /// + /// @param element element type (scalar or reference; nested vectors are not used by vortex) + public record Vector(FieldType element) implements FieldType { + } + + /// Reference to a declared table, struct, enum or union. Resolved post-parse. + /// + /// @param typeName the textual type reference as written + public record Ref(String typeName) implements FieldType { + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java new file mode 100644 index 000000000..864888671 --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java @@ -0,0 +1,730 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/// Emits MemorySegment-native FlatBuffers reader/builder classes from a [TypeRegistry]. +/// +/// One `.java` file per declaration: +/// - each `table` becomes a `final class extends FbsTable` with read accessors plus +/// `createX`/`startX`/`addX`/`endX`/`finishXBuffer` builders; +/// - each `struct` becomes a `final class extends FbsStruct` with inline-offset accessors +/// and an inline `createX` builder; +/// - each `enum` becomes a constants holder over its underlying integer type; +/// - each `union` becomes a discriminator-constants holder (NONE = 0, members from 1). +/// +/// The wire layout is the standard FlatBuffers binary format — see [FbsTable]/[FbsBuilder]. +public final class CodeGen { + + private static final String RUNTIME_PACKAGE = "io.github.dfa1.vortex.fbsrt"; + + /// Prefix applied to every emitted Java class name so the generated wire-format + /// classes (`Array`, `DType`, `Bool`, `List`, …) do not collide with project or JDK + /// types of the same name. Schema files keep the upstream (unprefixed) names; the + /// prefix lives here, mirroring proto-gen's `JAVA_NAME_PREFIX`. + private static final String JAVA_NAME_PREFIX = "Fbs"; + + private static final String GENERATED_ANNOTATION = "@Generated(\"io.github.dfa1.vortex.fbsgen.CodeGen\")\n"; + + private final TypeRegistry registry; + + /// @param registry resolved schema model + public CodeGen(TypeRegistry registry) { + this.registry = registry; + } + + /// Emits every declaration to `outDir`, one file per type. + /// + /// @param outDir output directory (the package directory) + /// @throws IOException on filesystem errors + public void emit(Path outDir) throws IOException { + Files.createDirectories(outDir); + for (Ast.Decl decl : registry.all()) { + String src = switch (decl) { + case Ast.TableDecl t -> emitTable(t); + case Ast.StructDecl s -> emitStruct(s); + case Ast.EnumDecl e -> emitEnum(e); + case Ast.UnionDecl u -> emitUnion(u); + }; + Files.writeString(outDir.resolve(className(decl.name()) + ".java"), src); + } + } + + private String preamble(String importBlock) { + StringBuilder sb = new StringBuilder(); + sb.append("// Generated by vortex-fbs-gen. Do not edit.\n\n"); + sb.append("package ").append(registry.javaPackage()).append(";\n\n"); + sb.append(importBlock); + sb.append("import javax.annotation.processing.Generated;\n\n"); + return sb.toString(); + } + + /// Maps a schema type name to its generated Java class name (prefixed). + /// + /// @param schemaName the name as written in the `.fbs` schema + /// @return the Java class name + private static String className(String schemaName) { + return JAVA_NAME_PREFIX + schemaName; + } + + // ---- tables --------------------------------------------------------------- + + private String emitTable(Ast.TableDecl table) { + String name = className(table.name()); + StringBuilder sb = new StringBuilder(); + sb.append(preamble("import " + RUNTIME_PACKAGE + ".FbsBuilder;\nimport " + RUNTIME_PACKAGE + + ".FbsTable;\nimport java.lang.foreign.MemorySegment;\n")); + sb.append("/// Reader and builder for the `").append(table.name()).append("` FlatBuffers table.\n"); + sb.append(GENERATED_ANNOTATION); + sb.append("public final class ").append(name).append(" extends FbsTable {\n\n"); + + sb.append(" /// Positions a reader at the root table of a finished buffer.\n"); + sb.append(" /// @param seg the buffer\n"); + sb.append(" /// @return a reader positioned at the root\n"); + sb.append(" public static ").append(name).append(" getRootAs").append(name) + .append("(MemorySegment seg) {\n"); + sb.append(" return new ").append(name).append("().assign(seg, FbsTable.rootPosition(seg, 0));\n"); + sb.append(" }\n\n"); + + sb.append(" /// Positions this reader at a table.\n"); + sb.append(" /// @param seg the buffer\n"); + sb.append(" /// @param position the table position\n"); + sb.append(" /// @return this\n"); + sb.append(" public ").append(name).append(" assign(MemorySegment seg, long position) {\n"); + sb.append(" init(seg, position);\n"); + sb.append(" return this;\n"); + sb.append(" }\n"); + + int vtableIndex = 0; + for (Ast.Field field : table.fields()) { + vtableIndex = emitTableField(sb, field, vtableIndex); + } + emitTableBuilder(sb, table); + sb.append("}\n"); + return sb.toString(); + } + + /// Emits accessors for one table field; returns the next vtable index (a union + /// consumes two slots: discriminator + value). + private int emitTableField(StringBuilder sb, Ast.Field field, int vtableIndex) { + String accessor = camelCase(field.name()); + Ast.FieldType type = field.type(); + int vt = 4 + vtableIndex * 2; + + if (type instanceof Ast.Scalar scalar) { + emitScalarTableField(sb, accessor, scalar, vt, field.defaultValue().orElse(null)); + return vtableIndex + 1; + } + if (type instanceof Ast.Vector vector) { + emitVectorField(sb, accessor, vector.element(), vt); + return vtableIndex + 1; + } + Ast.Ref ref = (Ast.Ref) type; + Ast.Decl target = registry.resolve(ref); + return switch (target) { + case Ast.EnumDecl e -> { + emitScalarTableField(sb, accessor, e.underlying(), vt, field.defaultValue().orElse(null)); + yield vtableIndex + 1; + } + case Ast.UnionDecl u -> { + emitUnionField(sb, accessor, vt); + yield vtableIndex + 2; + } + case Ast.TableDecl t -> { + emitObjectField(sb, accessor, className(t.name()), vt, true); + yield vtableIndex + 1; + } + case Ast.StructDecl s -> { + emitObjectField(sb, accessor, className(s.name()), vt, false); + yield vtableIndex + 1; + } + }; + } + + private void emitScalarTableField(StringBuilder sb, String accessor, Ast.Scalar scalar, int vt, String defaultValue) { + ScalarIo io = scalarIo(scalar); + String read = io.tableRead(); + sb.append("\n /// @return the `").append(accessor).append("` field\n"); + if (scalar == Ast.Scalar.BOOL) { + boolean def = "true".equals(defaultValue); + sb.append(" public boolean ").append(accessor).append("() {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return o != 0 ? readByte(pos + o) != 0 : ").append(def).append(";\n"); + sb.append(" }\n"); + } else { + String def = defaultLiteral(io, defaultValue); + sb.append(" public ").append(io.javaType).append(" ").append(accessor).append("() {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return o != 0 ? ").append(read).append(" : ").append(def).append(";\n"); + sb.append(" }\n"); + } + if ("null".equals(defaultValue)) { + sb.append("\n /// @return whether the optional `").append(accessor).append("` field is present\n"); + sb.append(" public boolean has").append(capitalize(accessor)).append("() {\n"); + sb.append(" return fieldOffset(").append(vt).append(") != 0;\n"); + sb.append(" }\n"); + } + } + + private void emitObjectField(StringBuilder sb, String accessor, String typeName, int vt, boolean indirect) { + String target = indirect ? "indirect(pos + o)" : "pos + o"; + sb.append("\n /// @return the `").append(accessor).append("` child, or null if absent\n"); + sb.append(" public ").append(typeName).append(" ").append(accessor).append("() {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return o != 0 ? new ").append(typeName).append("().assign(seg, ").append(target).append(") : null;\n"); + sb.append(" }\n"); + } + + private void emitUnionField(StringBuilder sb, String accessor, int vtType) { + int vtValue = vtType + 2; + sb.append("\n /// @return the union discriminator for `").append(accessor).append("` (0 = none)\n"); + sb.append(" public int ").append(accessor).append("Type() {\n"); + sb.append(" int o = fieldOffset(").append(vtType).append(");\n"); + sb.append(" return o != 0 ? readByte(pos + o) & 0xFF : 0;\n"); + sb.append(" }\n"); + sb.append("\n /// Projects the selected `").append(accessor).append("` member onto the given reader.\n"); + sb.append(" /// @param obj a reader for the expected member type\n"); + sb.append(" /// @param the member reader type\n"); + sb.append(" /// @return the positioned reader, or null if the union is absent\n"); + sb.append(" public T ").append(accessor).append("(T obj) {\n"); + sb.append(" int o = fieldOffset(").append(vtValue).append(");\n"); + sb.append(" return o != 0 ? locate(obj, unionMemberPosition(o)) : null;\n"); + sb.append(" }\n"); + } + + private void emitVectorField(StringBuilder sb, String accessor, Ast.FieldType element, int vt) { + sb.append("\n /// @return the length of the `").append(accessor).append("` vector\n"); + sb.append(" public int ").append(accessor).append("Length() {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return o != 0 ? vectorLength(o) : 0;\n"); + sb.append(" }\n"); + + if (element instanceof Ast.Scalar scalar) { + emitScalarVectorElements(sb, accessor, scalar, vt); + return; + } + Ast.Ref ref = (Ast.Ref) element; + Ast.Decl target = registry.resolve(ref); + switch (target) { + case Ast.EnumDecl e -> emitScalarVectorElements(sb, accessor, e.underlying(), vt); + case Ast.TableDecl t -> emitObjectVectorElement(sb, accessor, className(t.name()), vt, 4, true); + case Ast.StructDecl s -> emitObjectVectorElement(sb, accessor, className(s.name()), vt, registry.layoutOf(s).size(), false); + case Ast.UnionDecl u -> throw new FbsParseException("vectors of unions are not supported ('" + accessor + "')"); + } + } + + private void emitScalarVectorElements(StringBuilder sb, String accessor, Ast.Scalar scalar, int vt) { + if (scalar == Ast.Scalar.STRING) { + sb.append("\n /// @param j element index\n /// @return the j-th `").append(accessor).append("` string\n"); + sb.append(" public String ").append(accessor).append("(int j) {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return readStringAt(vectorElements(o) + (long) j * 4);\n"); + sb.append(" }\n"); + return; + } + ScalarIo io = scalarIo(scalar); + int size = TypeRegistry.sizeOf(scalar); + sb.append("\n /// @param j element index\n /// @return the j-th `").append(accessor).append("` element\n"); + sb.append(" public ").append(io.javaType).append(" ").append(accessor).append("(int j) {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return ").append(io.vectorRead(size)).append(";\n"); + sb.append(" }\n"); + if (scalar == Ast.Scalar.UINT8 || scalar == Ast.Scalar.INT8) { + sb.append("\n /// @return a zero-copy slice of the `").append(accessor).append("` byte vector, or null if absent\n"); + sb.append(" public MemorySegment ").append(accessor).append("AsSegment() {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return o != 0 ? byteVector(o) : null;\n"); + sb.append(" }\n"); + } + } + + private void emitObjectVectorElement(StringBuilder sb, String accessor, String typeName, int vt, int stride, boolean indirect) { + String base = "vectorElements(o) + (long) j * " + stride; + String target = indirect ? "indirect(" + base + ")" : base; + sb.append("\n /// @param j element index\n /// @return the j-th `").append(accessor).append("` element\n"); + sb.append(" public ").append(typeName).append(" ").append(accessor).append("(int j) {\n"); + sb.append(" int o = fieldOffset(").append(vt).append(");\n"); + sb.append(" return new ").append(typeName).append("().assign(seg, ").append(target).append(");\n"); + sb.append(" }\n"); + } + + // ---- structs -------------------------------------------------------------- + + private String emitStruct(Ast.StructDecl struct) { + String name = className(struct.name()); + TypeRegistry.StructLayout layout = registry.layoutOf(struct); + StringBuilder sb = new StringBuilder(); + sb.append(preamble("import " + RUNTIME_PACKAGE + ".FbsBuilder;\nimport " + RUNTIME_PACKAGE + + ".FbsStruct;\nimport java.lang.foreign.MemorySegment;\n")); + sb.append("/// Reader and builder for the `").append(struct.name()).append("` FlatBuffers struct (inline, ") + .append(layout.size()).append(" bytes).\n"); + sb.append(GENERATED_ANNOTATION); + sb.append("public final class ").append(name).append(" extends FbsStruct {\n\n"); + sb.append(" /// Positions this reader at a struct.\n"); + sb.append(" /// @param seg the buffer\n"); + sb.append(" /// @param position the struct position\n"); + sb.append(" /// @return this\n"); + sb.append(" public ").append(name).append(" assign(MemorySegment seg, long position) {\n"); + sb.append(" init(seg, position);\n"); + sb.append(" return this;\n"); + sb.append(" }\n"); + + for (int i = 0; i < struct.fields().size(); i++) { + Ast.Field field = struct.fields().get(i); + int off = layout.fieldOffsets()[i]; + Ast.Scalar scalar = structFieldScalar(field); + emitScalarStructField(sb, camelCase(field.name()), scalar, off); + } + emitStructBuilder(sb, struct, layout); + sb.append("}\n"); + return sb.toString(); + } + + private Ast.Scalar structFieldScalar(Ast.Field field) { + if (field.type() instanceof Ast.Scalar s) { + return s; + } + return ((Ast.EnumDecl) registry.resolve((Ast.Ref) field.type())).underlying(); + } + + private void emitScalarStructField(StringBuilder sb, String accessor, Ast.Scalar scalar, int off) { + ScalarIo io = scalarIo(scalar); + sb.append("\n /// @return the `").append(accessor).append("` field\n"); + if (scalar == Ast.Scalar.BOOL) { + sb.append(" public boolean ").append(accessor).append("() {\n"); + sb.append(" return readByte(").append(off).append(") != 0;\n"); + sb.append(" }\n"); + } else { + sb.append(" public ").append(io.javaType).append(" ").append(accessor).append("() {\n"); + sb.append(" return ").append(io.structRead(off)).append(";\n"); + sb.append(" }\n"); + } + } + + // ---- enums & unions ------------------------------------------------------- + + private String emitEnum(Ast.EnumDecl e) { + String constType = enumConstType(e.underlying()); + StringBuilder sb = new StringBuilder(); + String name = className(e.name()); + sb.append(preamble("")); + sb.append("/// Constants for the `").append(e.name()).append("` FlatBuffers enum.\n"); + sb.append(GENERATED_ANNOTATION); + sb.append("public final class ").append(name).append(" {\n\n"); + sb.append(" private ").append(name).append("() {\n }\n"); + long next = 0; + for (Ast.EnumVal v : e.values()) { + long value = v.number().isPresent() ? v.number().getAsInt() : next; + next = value + 1; + sb.append("\n /// `").append(v.name()).append("` = ").append(value).append("\n"); + sb.append(" public static final ").append(constType).append(" ").append(v.name()) + .append(" = ").append(literal(constType, value)).append(";\n"); + } + sb.append("}\n"); + return sb.toString(); + } + + private String emitUnion(Ast.UnionDecl u) { + StringBuilder sb = new StringBuilder(); + String name = className(u.name()); + sb.append(preamble("")); + sb.append("/// Discriminator constants for the `").append(u.name()).append("` FlatBuffers union.\n"); + sb.append(GENERATED_ANNOTATION); + sb.append("public final class ").append(name).append(" {\n\n"); + sb.append(" private ").append(name).append("() {\n }\n"); + sb.append("\n /// No member selected.\n"); + sb.append(" public static final byte NONE = 0;\n"); + long next = 1; + for (Ast.UnionMember m : u.members()) { + long value = m.number().isPresent() ? m.number().getAsInt() : next; + next = value + 1; + String member = className(simpleName(m.typeName())); + sb.append("\n /// Member `").append(simpleName(m.typeName())).append("` = ").append(value).append("\n"); + sb.append(" public static final byte ").append(member).append(" = ").append(literal("byte", value)).append(";\n"); + } + sb.append("}\n"); + return sb.toString(); + } + + // ---- table builders ------------------------------------------------------- + + private void emitTableBuilder(StringBuilder sb, Ast.TableDecl table) { + String name = className(table.name()); + List params = new ArrayList<>(); + List adds = new ArrayList<>(); + int slots = 0; + for (Ast.Field field : table.fields()) { + slots = collectBuilderField(sb, field, slots, params, adds); + } + // Match flatc's createX ordering: add fields by descending alignment, and within + // an alignment group in reverse declaration order. This keeps the data region gap-free + // and makes our output byte-identical to flatc. + // + // Limitation: an inline struct field (addStruct) must be the most-recently-written + // object when it is slotted (FbsBuilder.addStruct enforces x == offset()). This holds + // for vortex today because structs (Buffer, SegmentSpec) only ever appear as VECTOR + // elements, never as inline table fields. If a future schema adds an inline struct + // field that shares an alignment group with a scalar sorted ahead of it, this ordering + // could place the scalar between the struct write and its slot and trip that guard. + // Revisit the ordering (emit struct fields last within their group) if that ever lands. + adds.sort((x, y) -> x.align != y.align ? Integer.compare(y.align, x.align) : Integer.compare(y.order, x.order)); + + sb.append("\n /// Builds a `").append(name).append("` table.\n"); + sb.append(" /// @param b the builder\n"); + for (String p : params) { + sb.append(" /// @param ").append(p.substring(p.indexOf(' ') + 1)).append(" field value\n"); + } + sb.append(" /// @return the table offset\n"); + sb.append(" public static int create").append(name).append("(FbsBuilder b"); + for (String p : params) { + sb.append(", ").append(p); + } + sb.append(") {\n"); + sb.append(" b.startTable(").append(slots).append(");\n"); + for (AddEntry add : adds) { + sb.append(" ").append(add.statement).append("\n"); + } + sb.append(" return b.endTable();\n }\n"); + + sb.append("\n /// Begins a `").append(name).append("` table.\n"); + sb.append(" /// @param b the builder\n"); + sb.append(" public static void start").append(name).append("(FbsBuilder b) {\n"); + sb.append(" b.startTable(").append(slots).append(");\n }\n"); + + sb.append("\n /// Finishes a `").append(name).append("` table.\n"); + sb.append(" /// @param b the builder\n /// @return the table offset\n"); + sb.append(" public static int end").append(name).append("(FbsBuilder b) {\n"); + sb.append(" return b.endTable();\n }\n"); + + sb.append("\n /// Finishes the buffer with a `").append(name).append("` root.\n"); + sb.append(" /// @param b the builder\n /// @param offset the root table offset\n"); + sb.append(" public static void finish").append(name).append("Buffer(FbsBuilder b, int offset) {\n"); + sb.append(" b.finish(offset);\n }\n"); + } + + /// One add-call emitted inside `createX`, tagged with its alignment and declaration + /// order so the calls can be reordered to match flatc. + private record AddEntry(String statement, int align, int order) { + } + + /// Emits per-field `add*` methods (and vector helpers), and records the createX + /// parameter list and add-call list; returns the next slot index. + private int collectBuilderField(StringBuilder sb, Ast.Field field, int slot, List params, List adds) { + String accessor = camelCase(field.name()); + String capital = capitalize(accessor); + Ast.FieldType type = field.type(); + + if (type instanceof Ast.Vector vector) { + params.add("int " + accessor + "Offset"); + adds.add(new AddEntry("b.addOffset(" + slot + ", " + accessor + "Offset, 0);", 4, adds.size())); + emitAddOffsetMethod(sb, accessor, capital, slot); + emitVectorBuilders(sb, accessor, capital, vector.element()); + return slot + 1; + } + if (type instanceof Ast.Ref ref) { + Ast.Decl target = registry.resolve(ref); + if (target instanceof Ast.UnionDecl) { + params.add("byte " + accessor + "Type"); + params.add("int " + accessor + "Offset"); + adds.add(new AddEntry("b.addByte(" + slot + ", " + accessor + "Type, 0);", 1, adds.size())); + adds.add(new AddEntry("b.addOffset(" + (slot + 1) + ", " + accessor + "Offset, 0);", 4, adds.size())); + emitAddByteMethod(sb, accessor + "Type", capital + "Type", slot); + emitAddOffsetMethod(sb, accessor, capital, slot + 1); + return slot + 2; + } + if (target instanceof Ast.TableDecl) { + params.add("int " + accessor + "Offset"); + adds.add(new AddEntry("b.addOffset(" + slot + ", " + accessor + "Offset, 0);", 4, adds.size())); + emitAddOffsetMethod(sb, accessor, capital, slot); + return slot + 1; + } + if (target instanceof Ast.StructDecl s) { + int align = registry.layoutOf(s).alignment(); + params.add("int " + accessor + "Offset"); + adds.add(new AddEntry("b.addStruct(" + slot + ", " + accessor + "Offset, 0);", align, adds.size())); + return slot + 1; + } + // enum -> underlying scalar + Ast.Scalar underlying = ((Ast.EnumDecl) target).underlying(); + emitScalarBuilderField(sb, accessor, capital, underlying, slot, field.defaultValue().orElse(null), params, adds); + return slot + 1; + } + Ast.Scalar scalar = (Ast.Scalar) type; + if (scalar == Ast.Scalar.STRING) { + params.add("int " + accessor + "Offset"); + adds.add(new AddEntry("b.addOffset(" + slot + ", " + accessor + "Offset, 0);", 4, adds.size())); + emitAddOffsetMethod(sb, accessor, capital, slot); + return slot + 1; + } + emitScalarBuilderField(sb, accessor, capital, scalar, slot, field.defaultValue().orElse(null), params, adds); + return slot + 1; + } + + private void emitScalarBuilderField(StringBuilder sb, String accessor, String capital, Ast.Scalar scalar, + int slot, String defaultValue, List params, List adds) { + ScalarBuild sbuild = scalarBuild(scalar); + String paramType = scalarIo(scalar).javaType; + String def = buildDefault(sbuild.method, defaultValue); + String value = sbuild.cast + accessor; + params.add(paramType + " " + accessor); + adds.add(new AddEntry("b." + sbuild.method + "(" + slot + ", " + value + ", " + def + ");", TypeRegistry.sizeOf(scalar), adds.size())); + sb.append("\n /// Sets the `").append(accessor).append("` field.\n"); + sb.append(" /// @param b the builder\n /// @param ").append(accessor).append(" value\n"); + sb.append(" public static void add").append(capital).append("(FbsBuilder b, ").append(paramType) + .append(" ").append(accessor).append(") {\n"); + sb.append(" b.").append(sbuild.method).append("(").append(slot).append(", ").append(value) + .append(", ").append(def).append(");\n }\n"); + } + + private void emitAddOffsetMethod(StringBuilder sb, String accessor, String capital, int slot) { + sb.append("\n /// Sets the `").append(accessor).append("` offset field.\n"); + sb.append(" /// @param b the builder\n /// @param offset the referenced offset\n"); + sb.append(" public static void add").append(capital).append("(FbsBuilder b, int offset) {\n"); + sb.append(" b.addOffset(").append(slot).append(", offset, 0);\n }\n"); + } + + private void emitAddByteMethod(StringBuilder sb, String accessor, String capital, int slot) { + sb.append("\n /// Sets the `").append(accessor).append("` field.\n"); + sb.append(" /// @param b the builder\n /// @param value the value\n"); + sb.append(" public static void add").append(capital).append("(FbsBuilder b, byte value) {\n"); + sb.append(" b.addByte(").append(slot).append(", value, 0);\n }\n"); + } + + private void emitVectorBuilders(StringBuilder sb, String accessor, String capital, Ast.FieldType element) { + if (element instanceof Ast.Scalar scalar && (scalar == Ast.Scalar.UINT8 || scalar == Ast.Scalar.INT8)) { + sb.append("\n /// Creates the `").append(accessor).append("` byte vector.\n"); + sb.append(" /// @param b the builder\n /// @param data the bytes\n /// @return the vector offset\n"); + sb.append(" public static int create").append(capital).append("Vector(FbsBuilder b, byte[] data) {\n"); + sb.append(" return b.createByteVector(data);\n }\n"); + emitStartVector(sb, accessor, capital, 1, 1); + return; + } + if (element instanceof Ast.Scalar scalar && scalar != Ast.Scalar.STRING) { + VectorElem ve = scalarVectorElem(scalar); + sb.append("\n /// Creates the `").append(accessor).append("` vector.\n"); + sb.append(" /// @param b the builder\n /// @param data the elements\n /// @return the vector offset\n"); + sb.append(" public static int create").append(capital).append("Vector(FbsBuilder b, ").append(ve.arrayType).append(" data) {\n"); + sb.append(" b.startVector(").append(ve.size).append(", data.length, ").append(ve.size).append(");\n"); + sb.append(" for (int i = data.length - 1; i >= 0; i--) {\n"); + sb.append(" b.").append(ve.add).append("(").append(ve.cast).append("data[i]);\n }\n"); + sb.append(" return b.endVector();\n }\n"); + emitStartVector(sb, accessor, capital, ve.size, ve.size); + return; + } + // string or table-ref element -> offset vector; struct-ref element -> start only + if (element instanceof Ast.Scalar || isOffsetElement(element)) { + sb.append("\n /// Creates the `").append(accessor).append("` offset vector.\n"); + sb.append(" /// @param b the builder\n /// @param data element offsets\n /// @return the vector offset\n"); + sb.append(" public static int create").append(capital).append("Vector(FbsBuilder b, int[] data) {\n"); + sb.append(" b.startVector(4, data.length, 4);\n"); + sb.append(" for (int i = data.length - 1; i >= 0; i--) {\n"); + sb.append(" b.addOffset(data[i]);\n }\n"); + sb.append(" return b.endVector();\n }\n"); + emitStartVector(sb, accessor, capital, 4, 4); + return; + } + // struct element vector: only a start helper (elements written inline) + TypeRegistry.StructLayout layout = registry.layoutOf((Ast.StructDecl) registry.resolve((Ast.Ref) element)); + emitStartVector(sb, accessor, capital, layout.size(), layout.alignment()); + } + + private boolean isOffsetElement(Ast.FieldType element) { + return element instanceof Ast.Ref ref && registry.resolve(ref) instanceof Ast.TableDecl; + } + + private void emitStartVector(StringBuilder sb, String accessor, String capital, int elemSize, int alignment) { + sb.append("\n /// Begins the `").append(accessor).append("` vector.\n"); + sb.append(" /// @param b the builder\n /// @param numElems element count\n"); + sb.append(" public static void start").append(capital).append("Vector(FbsBuilder b, int numElems) {\n"); + sb.append(" b.startVector(").append(elemSize).append(", numElems, ").append(alignment).append(");\n }\n"); + } + + // ---- struct builder ------------------------------------------------------- + + private void emitStructBuilder(StringBuilder sb, Ast.StructDecl struct, TypeRegistry.StructLayout layout) { + String name = className(struct.name()); + List fields = struct.fields(); + sb.append("\n /// Writes an inline `").append(name).append("` struct.\n"); + sb.append(" /// @param b the builder\n"); + for (Ast.Field f : fields) { + sb.append(" /// @param ").append(camelCase(f.name())).append(" field value\n"); + } + sb.append(" /// @return the struct offset\n"); + sb.append(" public static int create").append(name).append("(FbsBuilder b"); + for (Ast.Field f : fields) { + sb.append(", ").append(scalarIo(structFieldScalar(f)).javaType).append(" ").append(camelCase(f.name())); + } + sb.append(") {\n"); + sb.append(" b.prep(").append(layout.alignment()).append(", ").append(layout.size()).append(");\n"); + int cursor = layout.size(); + for (int i = fields.size() - 1; i >= 0; i--) { + Ast.Field f = fields.get(i); + Ast.Scalar scalar = structFieldScalar(f); + int off = layout.fieldOffsets()[i]; + int gap = cursor - (off + TypeRegistry.sizeOf(scalar)); + if (gap > 0) { + sb.append(" b.pad(").append(gap).append(");\n"); + } + ScalarBuild sbuild = scalarBuild(scalar); + sb.append(" b.").append(sbuild.put).append("(").append(sbuild.cast).append(camelCase(f.name())).append(");\n"); + cursor = off; + } + if (cursor > 0) { + sb.append(" b.pad(").append(cursor).append(");\n"); + } + sb.append(" return b.offset();\n }\n"); + } + + private record ScalarBuild(String method, String put, String cast) { + } + + private static ScalarBuild scalarBuild(Ast.Scalar scalar) { + return switch (scalar) { + case BOOL -> new ScalarBuild("addBoolean", "putByte", ""); + case INT8, UINT8 -> new ScalarBuild("addByte", "putByte", "(byte) "); + case INT16, UINT16 -> new ScalarBuild("addShort", "putShort", "(short) "); + case INT32 -> new ScalarBuild("addInt", "putInt", ""); + case UINT32 -> new ScalarBuild("addInt", "putInt", "(int) "); + case INT64, UINT64 -> new ScalarBuild("addLong", "putLong", ""); + case FLOAT32 -> new ScalarBuild("addFloat", "putFloat", ""); + case FLOAT64 -> new ScalarBuild("addDouble", "putDouble", ""); + case STRING -> new ScalarBuild("addOffset", "putInt", ""); + }; + } + + /// Default literal for an `addX` builder call, typed to match the add method + /// (e.g. `addInt` takes an `int` default even for an unsigned 32-bit field). + private static String buildDefault(String addMethod, String defaultValue) { + boolean has = defaultValue != null && !"null".equals(defaultValue); + return switch (addMethod) { + case "addBoolean" -> Boolean.toString("true".equals(defaultValue)); + case "addLong" -> (has ? defaultValue : "0") + "L"; + case "addFloat" -> (has ? defaultValue : "0") + "f"; + case "addDouble" -> has ? defaultValue : "0.0"; + default -> has ? defaultValue : "0"; + }; + } + + private record VectorElem(String arrayType, String add, String cast, int size) { + } + + private static VectorElem scalarVectorElem(Ast.Scalar scalar) { + return switch (scalar) { + case INT16, UINT16 -> new VectorElem("int[]", "addShort", "(short) ", 2); + case INT32 -> new VectorElem("int[]", "addInt", "", 4); + case UINT32 -> new VectorElem("long[]", "addInt", "(int) ", 4); + case INT64, UINT64 -> new VectorElem("long[]", "addLong", "", 8); + case FLOAT32 -> new VectorElem("float[]", "addFloat", "", 4); + case FLOAT64 -> new VectorElem("double[]", "addDouble", "", 8); + default -> throw new FbsParseException("unsupported scalar vector element: " + scalar); + }; + } + + // ---- scalar I/O model ----------------------------------------------------- + + /// Read-expression model for one scalar. A read is composed structurally as + /// `prefix + method(position) + suffix` (e.g. `(int) ` + `readShort(at)` + ``, or + /// `readInt(at)` + ` & 0xFFFFFFFFL`), so the same scalar reads correctly at any + /// position — table field (`pos + o`), vector element, or struct offset — without + /// rewriting a literal string. + /// + /// @param javaType accessor return type + /// @param readPrefix expression prefix (e.g. a widening cast), or empty + /// @param readMethod FbsTable/FbsStruct reader method name + /// @param readSuffix expression suffix (e.g. an unsigned mask or `!= 0`), or empty + /// @param zero default/absent value literal + private record ScalarIo(String javaType, String readPrefix, String readMethod, String readSuffix, String zero) { + /// @param at the absolute/relative byte-position expression to read at + /// @return the read expression for that position + String read(String at) { + return readPrefix + readMethod + "(" + at + ")" + readSuffix; + } + + String tableRead() { + return read("pos + o"); + } + + String vectorRead(int size) { + return "o != 0 ? " + read("vectorElements(o) + (long) j * " + size) + " : " + zero; + } + + String structRead(int off) { + return read(Integer.toString(off)); + } + } + + private static ScalarIo scalarIo(Ast.Scalar scalar) { + return switch (scalar) { + case BOOL -> new ScalarIo("boolean", "", "readByte", " != 0", "false"); + case INT8 -> new ScalarIo("int", "(int) ", "readByte", "", "0"); + case UINT8 -> new ScalarIo("int", "", "readByte", " & 0xFF", "0"); + case INT16 -> new ScalarIo("int", "(int) ", "readShort", "", "0"); + case UINT16 -> new ScalarIo("int", "", "readShort", " & 0xFFFF", "0"); + case INT32 -> new ScalarIo("int", "", "readInt", "", "0"); + case UINT32 -> new ScalarIo("long", "", "readInt", " & 0xFFFFFFFFL", "0L"); + case INT64 -> new ScalarIo("long", "", "readLong", "", "0L"); + case UINT64 -> new ScalarIo("long", "", "readLong", "", "0L"); + case FLOAT32 -> new ScalarIo("float", "", "readFloat", "", "0.0f"); + case FLOAT64 -> new ScalarIo("double", "", "readDouble", "", "0.0"); + case STRING -> new ScalarIo("String", "", "readStringAt", "", "null"); + }; + } + + private static String defaultLiteral(ScalarIo io, String defaultValue) { + if (defaultValue == null || "null".equals(defaultValue)) { + return io.zero; + } + return switch (io.javaType) { + case "long" -> defaultValue + "L"; + case "float" -> defaultValue + "f"; + default -> defaultValue; + }; + } + + private static String enumConstType(Ast.Scalar underlying) { + return switch (TypeRegistry.sizeOf(underlying)) { + case 1 -> "byte"; + case 2 -> "short"; + case 8 -> "long"; + default -> "int"; + }; + } + + private static String literal(String constType, long value) { + return switch (constType) { + case "byte" -> "(byte) " + value; + case "short" -> "(short) " + value; + case "long" -> value + "L"; + default -> Long.toString(value); + }; + } + + private static String simpleName(String qualified) { + int dot = qualified.lastIndexOf('.'); + return dot >= 0 ? qualified.substring(dot + 1) : qualified; + } + + private static String camelCase(String snake) { + StringBuilder sb = new StringBuilder(); + boolean upper = false; + for (int i = 0; i < snake.length(); i++) { + char c = snake.charAt(i); + if (c == '_') { + upper = sb.length() > 0; + } else if (upper) { + sb.append(Character.toUpperCase(c)); + upper = false; + } else { + sb.append(c); + } + } + return sb.isEmpty() ? snake : sb.toString(); + } + + private static String capitalize(String s) { + return s.isEmpty() ? s : Character.toUpperCase(s.charAt(0)) + s.substring(1); + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/FbsParseException.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/FbsParseException.java new file mode 100644 index 000000000..6d7b7f8c9 --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/FbsParseException.java @@ -0,0 +1,12 @@ +package io.github.dfa1.vortex.fbsgen; + +/// Thrown when a `.fbs` schema cannot be lexed or parsed. +public final class FbsParseException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /// @param message human-readable diagnostic, ideally with a 1-based line number + public FbsParseException(String message) { + super(message); + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Lexer.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Lexer.java new file mode 100644 index 000000000..0e576a987 --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Lexer.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.util.ArrayList; +import java.util.List; + +/// Tokenizes a FlatBuffers schema (`.fbs`) source file into a list of [Token]s. +/// Comments (`//` and `///` line, `/* ... */` block) and whitespace are dropped. +public final class Lexer { + + private final String src; + private int pos; + private int line; + + /// @param src raw `.fbs` source text + public Lexer(String src) { + this.src = src; + this.pos = 0; + this.line = 1; + } + + /// Lex the entire source and return the token stream, terminated by a single `EOF`. + /// @return immutable token list + public List tokenize() { + List out = new ArrayList<>(); + while (true) { + skipWhitespaceAndComments(); + if (pos >= src.length()) { + out.add(new Token(Token.Kind.EOF, "", line)); + return List.copyOf(out); + } + char c = src.charAt(pos); + if (isIdentStart(c)) { + out.add(readIdent()); + } else if (Character.isDigit(c) || (c == '-' && pos + 1 < src.length() && Character.isDigit(src.charAt(pos + 1)))) { + out.add(readNumber()); + } else if (c == '"') { + out.add(readString()); + } else { + out.add(readPunct()); + } + } + } + + private Token readIdent() { + int start = pos; + while (pos < src.length() && isIdentPart(src.charAt(pos))) { + pos++; + } + return new Token(Token.Kind.IDENT, src.substring(start, pos), line); + } + + private Token readNumber() { + int start = pos; + if (src.charAt(pos) == '-') { + pos++; + } + while (pos < src.length() && Character.isDigit(src.charAt(pos))) { + pos++; + } + return new Token(Token.Kind.INT_LITERAL, src.substring(start, pos), line); + } + + private Token readString() { + int startLine = line; + pos++; // skip opening " + StringBuilder sb = new StringBuilder(); + while (pos < src.length() && src.charAt(pos) != '"') { + char c = src.charAt(pos++); + if (c == '\\' && pos < src.length()) { + char esc = src.charAt(pos++); + sb.append(switch (esc) { + case 'n' -> '\n'; + case 't' -> '\t'; + case 'r' -> '\r'; + case '"' -> '"'; + case '\\' -> '\\'; + default -> esc; + }); + } else { + if (c == '\n') { + line++; + } + sb.append(c); + } + } + if (pos >= src.length()) { + throw new FbsParseException("unterminated string at line " + startLine); + } + pos++; // skip closing " + return new Token(Token.Kind.STRING_LITERAL, sb.toString(), startLine); + } + + private Token readPunct() { + char c = src.charAt(pos++); + Token.Kind kind = switch (c) { + case '{' -> Token.Kind.LBRACE; + case '}' -> Token.Kind.RBRACE; + case '(' -> Token.Kind.LPAREN; + case ')' -> Token.Kind.RPAREN; + case '[' -> Token.Kind.LBRACKET; + case ']' -> Token.Kind.RBRACKET; + case ':' -> Token.Kind.COLON; + case ';' -> Token.Kind.SEMICOLON; + case ',' -> Token.Kind.COMMA; + case '=' -> Token.Kind.EQUALS; + case '.' -> Token.Kind.DOT; + default -> throw new FbsParseException("unexpected character '" + c + "' at line " + line); + }; + return new Token(kind, String.valueOf(c), line); + } + + private void skipWhitespaceAndComments() { + while (pos < src.length()) { + char c = src.charAt(pos); + if (c == '\n') { + line++; + pos++; + } else if (Character.isWhitespace(c)) { + pos++; + } else if (c == '/' && pos + 1 < src.length() && src.charAt(pos + 1) == '/') { + pos += 2; + while (pos < src.length() && src.charAt(pos) != '\n') { + pos++; + } + } else if (c == '/' && pos + 1 < src.length() && src.charAt(pos + 1) == '*') { + pos += 2; + while (pos + 1 < src.length() && !(src.charAt(pos) == '*' && src.charAt(pos + 1) == '/')) { + if (src.charAt(pos) == '\n') { + line++; + } + pos++; + } + if (pos + 1 >= src.length()) { + throw new FbsParseException("unterminated block comment"); + } + pos += 2; + } else { + return; + } + } + } + + private static boolean isIdentStart(char c) { + return Character.isLetter(c) || c == '_'; + } + + private static boolean isIdentPart(char c) { + return Character.isLetterOrDigit(c) || c == '_'; + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Main.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Main.java new file mode 100644 index 000000000..3ba4390ba --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Main.java @@ -0,0 +1,50 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/// CLI entry for the build-time `.fbs` to MemorySegment-native Java code generator. +/// Invoked from the `regenerate-sources` Maven profile via `exec-maven-plugin`. +@SuppressWarnings("java:S106") // build-time code-gen CLI: progress goes to stdout +public final class Main { + + private Main() { + } + + /// Usage: `Main --out

[ ...]`. + /// @param args command-line arguments + /// @throws IOException on filesystem errors + public static void main(String[] args) throws IOException { + Path out = null; + List schemas = new ArrayList<>(); + for (int i = 0; i < args.length; i++) { + if ("--out".equals(args[i])) { + if (i + 1 >= args.length) { + usage(); + return; + } + out = Path.of(args[++i]); + } else { + schemas.add(Path.of(args[i])); + } + } + if (out == null || schemas.isEmpty()) { + usage(); + return; + } + List files = new ArrayList<>(schemas.size()); + for (Path p : schemas) { + files.add(new Parser(new Lexer(Files.readString(p)).tokenize()).parseFile()); + } + new CodeGen(new TypeRegistry(files)).emit(out); + System.out.println("fbsgen: wrote " + schemas.size() + " .fbs file(s) to " + out); + } + + private static void usage() { + System.err.println("usage: fbsgen --out [ ...]"); + System.exit(2); + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Parser.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Parser.java new file mode 100644 index 000000000..1f794fbeb --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Parser.java @@ -0,0 +1,259 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +/// Recursive-descent parser for the FlatBuffers schema (`.fbs`) subset used by vortex. +/// Accepts: `namespace`, `include`, `table`, `struct`, `enum`, `union`, `root_type`, +/// vector types `[T]`, the `(required)` attribute, scalar default values, and the +/// FlatBuffers scalar types (with width aliases such as `byte`/`ubyte`/`int`/`long`). +/// Rejects: `attribute` declarations, `rpc_service`, key/hash attributes, fixed-size +/// arrays, nested namespaces within a single file, and unions of non-table members. +public final class Parser { + + /// FlatBuffers scalar keywords (including width aliases) mapped to canonical scalars. + private static final Map SCALARS = Map.ofEntries( + Map.entry("bool", Ast.Scalar.BOOL), + Map.entry("byte", Ast.Scalar.INT8), + Map.entry("int8", Ast.Scalar.INT8), + Map.entry("ubyte", Ast.Scalar.UINT8), + Map.entry("uint8", Ast.Scalar.UINT8), + Map.entry("short", Ast.Scalar.INT16), + Map.entry("int16", Ast.Scalar.INT16), + Map.entry("ushort", Ast.Scalar.UINT16), + Map.entry("uint16", Ast.Scalar.UINT16), + Map.entry("int", Ast.Scalar.INT32), + Map.entry("int32", Ast.Scalar.INT32), + Map.entry("uint", Ast.Scalar.UINT32), + Map.entry("uint32", Ast.Scalar.UINT32), + Map.entry("long", Ast.Scalar.INT64), + Map.entry("int64", Ast.Scalar.INT64), + Map.entry("ulong", Ast.Scalar.UINT64), + Map.entry("uint64", Ast.Scalar.UINT64), + Map.entry("float", Ast.Scalar.FLOAT32), + Map.entry("float32", Ast.Scalar.FLOAT32), + Map.entry("double", Ast.Scalar.FLOAT64), + Map.entry("float64", Ast.Scalar.FLOAT64), + Map.entry("string", Ast.Scalar.STRING)); + + private final List tokens; + private int pos; + + /// @param tokens token stream produced by [Lexer] + public Parser(List tokens) { + this.tokens = tokens; + this.pos = 0; + } + + /// Parses one `.fbs` file. + /// @return the parsed schema AST + public Ast.SchemaFile parseFile() { + String namespace = ""; + List includes = new ArrayList<>(); + List decls = new ArrayList<>(); + List rootTypes = new ArrayList<>(); + + while (peek().kind() != Token.Kind.EOF) { + Token t = peek(); + if (t.kind() != Token.Kind.IDENT) { + throw new FbsParseException("expected top-level declaration at line " + t.line() + ", got " + t.kind()); + } + switch (t.text()) { + case "namespace" -> namespace = parseNamespace(); + case "include" -> includes.add(parseInclude()); + case "table" -> decls.add(parseTableOrStruct(true)); + case "struct" -> decls.add(parseTableOrStruct(false)); + case "enum" -> decls.add(parseEnum()); + case "union" -> decls.add(parseUnion()); + case "root_type" -> rootTypes.add(parseRootType()); + default -> throw new FbsParseException("unexpected top-level token '" + t.text() + "' at line " + t.line()); + } + } + return new Ast.SchemaFile(namespace, List.copyOf(includes), List.copyOf(decls), List.copyOf(rootTypes)); + } + + private String parseNamespace() { + expectIdent("namespace"); + String ns = parseQualifiedName(); + expect(Token.Kind.SEMICOLON); + return ns; + } + + private String parseInclude() { + expectIdent("include"); + Token path = expect(Token.Kind.STRING_LITERAL); + expect(Token.Kind.SEMICOLON); + return path.text(); + } + + private String parseRootType() { + expectIdent("root_type"); + String name = parseQualifiedName(); + expect(Token.Kind.SEMICOLON); + return name; + } + + private Ast.Decl parseTableOrStruct(boolean isTable) { + advance(); // 'table' | 'struct' + String name = expect(Token.Kind.IDENT).text(); + expect(Token.Kind.LBRACE); + List fields = new ArrayList<>(); + while (peek().kind() != Token.Kind.RBRACE) { + fields.add(parseField()); + } + expect(Token.Kind.RBRACE); + return isTable ? new Ast.TableDecl(name, List.copyOf(fields)) : new Ast.StructDecl(name, List.copyOf(fields)); + } + + private Ast.Field parseField() { + String name = expect(Token.Kind.IDENT).text(); + expect(Token.Kind.COLON); + Ast.FieldType type = parseType(); + Optional defaultValue = Optional.empty(); + if (peek().kind() == Token.Kind.EQUALS) { + advance(); + defaultValue = Optional.of(parseDefaultLiteral()); + } + boolean required = false; + if (peek().kind() == Token.Kind.LPAREN) { + required = parseAttributes(); + } + expect(Token.Kind.SEMICOLON); + return new Ast.Field(name, type, defaultValue, required); + } + + private Ast.FieldType parseType() { + if (peek().kind() == Token.Kind.LBRACKET) { + advance(); + Ast.FieldType element = parseType(); + expect(Token.Kind.RBRACKET); + return new Ast.Vector(element); + } + String name = parseQualifiedName(); + Ast.Scalar scalar = SCALARS.get(name); + return scalar != null ? scalar : new Ast.Ref(name); + } + + private String parseDefaultLiteral() { + Token t = peek(); + if (t.kind() == Token.Kind.INT_LITERAL || t.kind() == Token.Kind.IDENT) { + advance(); + return t.text(); + } + throw new FbsParseException("expected a default value at line " + t.line() + ", got " + t.kind()); + } + + /// Parses a `( attr [= value] [, ...] )` block. Only `required` is honored; + /// any other attribute is accepted and ignored. + private boolean parseAttributes() { + expect(Token.Kind.LPAREN); + boolean required = false; + while (peek().kind() != Token.Kind.RPAREN) { + String attr = expect(Token.Kind.IDENT).text(); + if ("required".equals(attr)) { + required = true; + } + if (peek().kind() == Token.Kind.EQUALS) { + advance(); + advance(); // attribute value (literal or string) — ignored + } + if (peek().kind() == Token.Kind.COMMA) { + advance(); + } + } + expect(Token.Kind.RPAREN); + return required; + } + + private Ast.Decl parseEnum() { + expectIdent("enum"); + String name = expect(Token.Kind.IDENT).text(); + expect(Token.Kind.COLON); + String underlyingName = parseQualifiedName(); + Ast.Scalar underlying = SCALARS.get(underlyingName); + if (underlying == null) { + throw new FbsParseException("enum '" + name + "' has non-scalar underlying type '" + underlyingName + "'"); + } + expect(Token.Kind.LBRACE); + List values = new ArrayList<>(); + while (peek().kind() != Token.Kind.RBRACE) { + String valName = expect(Token.Kind.IDENT).text(); + OptionalInt number = OptionalInt.empty(); + if (peek().kind() == Token.Kind.EQUALS) { + advance(); + number = OptionalInt.of(parseInt()); + } + values.add(new Ast.EnumVal(valName, number)); + if (peek().kind() == Token.Kind.COMMA) { + advance(); + } + } + expect(Token.Kind.RBRACE); + return new Ast.EnumDecl(name, underlying, List.copyOf(values)); + } + + private Ast.Decl parseUnion() { + expectIdent("union"); + String name = expect(Token.Kind.IDENT).text(); + expect(Token.Kind.LBRACE); + List members = new ArrayList<>(); + while (peek().kind() != Token.Kind.RBRACE) { + String typeName = parseQualifiedName(); + OptionalInt number = OptionalInt.empty(); + if (peek().kind() == Token.Kind.EQUALS) { + advance(); + number = OptionalInt.of(parseInt()); + } + members.add(new Ast.UnionMember(typeName, number)); + if (peek().kind() == Token.Kind.COMMA) { + advance(); + } + } + expect(Token.Kind.RBRACE); + return new Ast.UnionDecl(name, List.copyOf(members)); + } + + private String parseQualifiedName() { + StringBuilder sb = new StringBuilder(expect(Token.Kind.IDENT).text()); + while (peek().kind() == Token.Kind.DOT) { + advance(); + sb.append('.').append(expect(Token.Kind.IDENT).text()); + } + return sb.toString(); + } + + private int parseInt() { + Token t = expect(Token.Kind.INT_LITERAL); + try { + return Integer.parseInt(t.text()); + } catch (NumberFormatException e) { + throw new FbsParseException("invalid integer '" + t.text() + "' at line " + t.line()); + } + } + + private Token peek() { + return tokens.get(pos); + } + + private Token advance() { + return tokens.get(pos++); + } + + private Token expect(Token.Kind kind) { + Token t = peek(); + if (t.kind() != kind) { + throw new FbsParseException("expected " + kind + " at line " + t.line() + ", got " + t.kind() + " '" + t.text() + "'"); + } + return advance(); + } + + private void expectIdent(String text) { + Token t = expect(Token.Kind.IDENT); + if (!text.equals(t.text())) { + throw new FbsParseException("expected '" + text + "' at line " + t.line() + ", got '" + t.text() + "'"); + } + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Token.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Token.java new file mode 100644 index 000000000..af09fd8fd --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/Token.java @@ -0,0 +1,28 @@ +package io.github.dfa1.vortex.fbsgen; + +/// A token emitted by [Lexer]. +/// +/// @param kind token kind +/// @param text source text — meaningful for [Kind#IDENT], [Kind#INT_LITERAL], [Kind#STRING_LITERAL] +/// @param line 1-based source line number for diagnostics +public record Token(Kind kind, String text, int line) { + + /// Token classification. + public enum Kind { + IDENT, + INT_LITERAL, + STRING_LITERAL, + LBRACE, + RBRACE, + LPAREN, + RPAREN, + LBRACKET, + RBRACKET, + COLON, + SEMICOLON, + COMMA, + EQUALS, + DOT, + EOF + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/TypeRegistry.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/TypeRegistry.java new file mode 100644 index 000000000..d4a00ba51 --- /dev/null +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/TypeRegistry.java @@ -0,0 +1,121 @@ +package io.github.dfa1.vortex.fbsgen; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/// Resolves type references across the parsed `.fbs` files and exposes layout facts +/// (scalar widths, struct sizes/alignment) the generator needs. +/// +/// All vortex schemas share a single namespace, and type names are unique across +/// files, so references resolve by simple name (a leading namespace qualifier, if +/// written, is stripped to its last segment). +public final class TypeRegistry { + + private final String namespace; + private final Map byName = new HashMap<>(); + private final List decls = new ArrayList<>(); + + /// Builds a registry covering every declaration across `files`. + /// + /// @param files parsed schema files + public TypeRegistry(List files) { + String ns = ""; + for (Ast.SchemaFile f : files) { + if (!f.namespace().isEmpty()) { + ns = f.namespace(); + } + for (Ast.Decl d : f.decls()) { + byName.put(d.name(), d); + decls.add(d); + } + } + this.namespace = ns; + } + + /// @return the common Java package (derived from the schema namespace) + public String javaPackage() { + return namespace; + } + + /// @return all declarations across all files, in parse order + public List all() { + return List.copyOf(decls); + } + + /// Resolves a (possibly namespace-qualified) type reference. + /// + /// @param ref the textual reference + /// @return the declaration it names + public Ast.Decl resolve(Ast.Ref ref) { + String name = simpleName(ref.typeName()); + Ast.Decl d = byName.get(name); + if (d == null) { + throw new FbsParseException("unresolved type reference: '" + ref.typeName() + "'"); + } + return d; + } + + private static String simpleName(String qualified) { + int dot = qualified.lastIndexOf('.'); + return dot >= 0 ? qualified.substring(dot + 1) : qualified; + } + + /// @param scalar a scalar type + /// @return its wire size in bytes (a `string` is a 4-byte offset) + public static int sizeOf(Ast.Scalar scalar) { + return switch (scalar) { + case BOOL, INT8, UINT8 -> 1; + case INT16, UINT16 -> 2; + case INT32, UINT32, FLOAT32, STRING -> 4; + case INT64, UINT64, FLOAT64 -> 8; + }; + } + + /// Computes the inline byte offset of each field within a struct plus the struct's + /// total padded size, following FlatBuffers struct layout (each field aligned to its + /// own size; struct size padded to the maximum field alignment). + /// + /// @param struct the struct declaration + /// @return a record carrying per-field offsets (parallel to the field list) and total size + public StructLayout layoutOf(Ast.StructDecl struct) { + int[] offsets = new int[struct.fields().size()]; + int pos = 0; + int maxAlign = 1; + for (int i = 0; i < struct.fields().size(); i++) { + int size = fieldScalarSize(struct.fields().get(i)); + pos = align(pos, size); + offsets[i] = pos; + pos += size; + maxAlign = Math.max(maxAlign, size); + } + int total = align(pos, maxAlign); + return new StructLayout(offsets, total, maxAlign); + } + + /// Resolves the scalar backing a struct field — a direct scalar or an enum's + /// underlying type. Nested struct fields are not used by vortex and are rejected. + private int fieldScalarSize(Ast.Field field) { + Ast.FieldType t = field.type(); + if (t instanceof Ast.Scalar s) { + return sizeOf(s); + } + if (t instanceof Ast.Ref ref && resolve(ref) instanceof Ast.EnumDecl e) { + return sizeOf(e.underlying()); + } + throw new FbsParseException("unsupported struct field type for '" + field.name() + "'"); + } + + private static int align(int pos, int size) { + return (pos + size - 1) / size * size; + } + + /// Inline layout of a struct. + /// + /// @param fieldOffsets byte offset of each field, parallel to the struct's field list + /// @param size total padded struct size in bytes + /// @param alignment struct alignment (the maximum field alignment) + public record StructLayout(int[] fieldOffsets, int size, int alignment) { + } +} diff --git a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java new file mode 100644 index 000000000..5438bd852 --- /dev/null +++ b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java @@ -0,0 +1,119 @@ +package io.github.dfa1.vortex.fbsgen; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class CodeGenTest { + + @Test + void emitsOneFilePerDeclarationForAllVortexSchemas(@TempDir Path out) throws IOException { + // Given + When + generate(out); + + // Then — a representative file from each schema exists, with a package line + // before any import (the ordering bug guard). + assertThat(Files.exists(out.resolve("FbsArray.java"))).isTrue(); + assertThat(Files.exists(out.resolve("FbsDType.java"))).isTrue(); + assertThat(Files.exists(out.resolve("FbsFooter.java"))).isTrue(); + assertThat(Files.exists(out.resolve("FbsLayout.java"))).isTrue(); + String layout = Files.readString(out.resolve("FbsLayout.java")); + assertThat(layout.indexOf("package ")).isLessThan(layout.indexOf("import ")); + } + + @Test + void emitsScalarVectorAndChildAccessorsForTable() throws IOException { + // Given — Layout has u16/u64 scalars, a byte vector, a table-vector and a u32 vector. + String src = generateOne("FbsLayout.java"); + + // Then — accessors match the flatc read shapes (widened unsigned types). + assertThat(src).contains("extends FbsTable"); + assertThat(src).contains("public int encoding() {"); + assertThat(src).contains("public long rowCount() {"); + assertThat(src).contains("public MemorySegment metadataAsSegment() {"); + assertThat(src).contains("public int childrenLength() {"); + assertThat(src).contains("public FbsLayout children(int j) {"); + assertThat(src).contains("indirect(vectorElements(o) + (long) j * 4)"); + assertThat(src).contains("public long segments(int j) {"); + assertThat(src).contains("readInt(vectorElements(o) + (long) j * 4) & 0xFFFFFFFFL"); + } + + @Test + void emitsInlineOffsetsForStruct() throws IOException { + // Given — Buffer struct: padding u16 @0, alignment u8 @2, compression(enum u8) @3, + // length u32 @4. + String src = generateOne("FbsBuffer.java"); + + // Then + assertThat(src).contains("extends FbsStruct"); + assertThat(src).contains("8 bytes"); + assertThat(src).contains("public int padding() {\n return readShort(0) & 0xFFFF;"); + assertThat(src).contains("public int alignmentExponent() {\n return readByte(2) & 0xFF;"); + assertThat(src).contains("public int compression() {\n return readByte(3) & 0xFF;"); + assertThat(src).contains("public long length() {\n return readInt(4) & 0xFFFFFFFFL;"); + } + + @Test + void emitsDiscriminatorAndMemberPositionForUnionField() throws IOException { + // Given — DType has a single union field `type`. + String src = generateOne("FbsDType.java"); + + // Then — discriminator at vtable slot 4, value (member) projected from slot 6. + assertThat(src).contains("public int typeType() {"); + assertThat(src).contains("fieldOffset(4)"); + assertThat(src).contains("public T type(T obj) {"); + assertThat(src).contains("locate(obj, unionMemberPosition(o))"); + assertThat(src).contains("fieldOffset(6)"); + } + + @Test + void emitsUnionConstantsHolder() throws IOException { + // Given + String src = generateOne("FbsType.java"); + + // Then — NONE plus members with explicit discriminators (byte, like flatc). + assertThat(src).contains("public static final byte NONE = 0;"); + assertThat(src).contains("public static final byte FbsNull = (byte) 1;"); + assertThat(src).contains("public static final byte FbsUnion = (byte) 12;"); + } + + @Test + void emitsEnumConstantsOverUnderlyingType() throws IOException { + // Given — PType: uint8 with auto-incremented values. + String src = generateOne("FbsPType.java"); + + // Then — byte constants, auto-numbered from 0. + assertThat(src).contains("public static final byte U8 = (byte) 0;"); + assertThat(src).contains("public static final byte F64 = (byte) 10;"); + } + + private static void generate(Path out) throws IOException { + Path root = repoRoot(); + List files = List.of( + parse(root.resolve("core/src/main/flatbuffers/vortex-array/array.fbs")), + parse(root.resolve("core/src/main/flatbuffers/vortex-dtype/dtype.fbs")), + parse(root.resolve("core/src/main/flatbuffers/vortex-file/footer.fbs")), + parse(root.resolve("core/src/main/flatbuffers/vortex-layout/layout.fbs")) + ); + new CodeGen(new TypeRegistry(files)).emit(out); + } + + private static String generateOne(String fileName) throws IOException { + Path out = Files.createTempDirectory("fbsgen"); + generate(out); + return Files.readString(out.resolve(fileName)); + } + + private static Ast.SchemaFile parse(Path p) throws IOException { + return new Parser(new Lexer(Files.readString(p)).tokenize()).parseFile(); + } + + private static Path repoRoot() { + return Path.of("").toAbsolutePath().getParent(); + } +} diff --git a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/ParserTest.java b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/ParserTest.java new file mode 100644 index 000000000..f3ecf1d7a --- /dev/null +++ b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/ParserTest.java @@ -0,0 +1,165 @@ +package io.github.dfa1.vortex.fbsgen; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ParserTest { + + @Nested + class VortexSchemas { + + /// The four real vortex schemas must all parse — this is the spike's ground + /// truth: the parser has to cover everything actually used in the format, + /// not a synthetic subset. + @ParameterizedTest + @ValueSource(strings = { + "core/src/main/flatbuffers/vortex-array/array.fbs", + "core/src/main/flatbuffers/vortex-dtype/dtype.fbs", + "core/src/main/flatbuffers/vortex-file/footer.fbs", + "core/src/main/flatbuffers/vortex-layout/layout.fbs" + }) + void parsesEveryVortexSchema(String relPath) throws IOException { + // Given + Ast.SchemaFile sut = parse(repoRoot().resolve(relPath)); + + // Then — every schema shares this namespace and declares at least one type. + assertThat(sut.namespace()).isEqualTo("io.github.dfa1.vortex.fbs"); + assertThat(sut.decls()).isNotEmpty(); + } + + @Test + void parsesDtypeUnionAndTable() throws IOException { + // Given — dtype.fbs is the richest: enum-with-underlying, a 12-member union, + // and tables that reference each other recursively. + Ast.SchemaFile sut = parse(repoRoot().resolve("core/src/main/flatbuffers/vortex-dtype/dtype.fbs")); + + // Then — the underlying-typed enum is captured with all 11 PType values. + Ast.EnumDecl ptype = (Ast.EnumDecl) declNamed(sut, "PType"); + assertThat(ptype.underlying()).isEqualTo(Ast.Scalar.UINT8); + assertThat(ptype.values()).hasSize(11); + + // And — the union is captured with its 12 members and explicit discriminators. + Ast.UnionDecl type = (Ast.UnionDecl) declNamed(sut, "Type"); + assertThat(type.members()).hasSize(12); + assertThat(type.members().getFirst().typeName()).isEqualTo("Null"); + assertThat(type.members().getFirst().number()).hasValue(1); + + // And — the DType table holds a single union-typed field. + Ast.TableDecl dtype = (Ast.TableDecl) declNamed(sut, "DType"); + assertThat(dtype.fields()).singleElement() + .satisfies(f -> assertThat(((Ast.Ref) f.type()).typeName()).isEqualTo("Type")); + } + + @Test + void parsesArrayStructVectorAndDefaults() throws IOException { + // Given — array.fbs exercises a struct (inline), vectors, an enum field, + // and `bool = null` three-state defaults. + Ast.SchemaFile sut = parse(repoRoot().resolve("core/src/main/flatbuffers/vortex-array/array.fbs")); + + // Then — Buffer is a struct, and its compression field references the enum. + Ast.StructDecl buffer = (Ast.StructDecl) declNamed(sut, "Buffer"); + Ast.Field compression = fieldNamed(buffer.fields(), "compression"); + assertThat(((Ast.Ref) compression.type()).typeName()).isEqualTo("Compression"); + + // And — Array.buffers is a vector of Buffer. + Ast.TableDecl array = (Ast.TableDecl) declNamed(sut, "Array"); + Ast.Field buffers = fieldNamed(array.fields(), "buffers"); + assertThat(((Ast.Ref) ((Ast.Vector) buffers.type()).element()).typeName()).isEqualTo("Buffer"); + + // And — ArrayStats keeps the `= null` defaults verbatim for tri-state bools. + Ast.TableDecl stats = (Ast.TableDecl) declNamed(sut, "ArrayStats"); + assertThat(fieldNamed(stats.fields(), "is_sorted").defaultValue()).hasValue("null"); + } + + @Test + void capturesIncludesAndRootTypes() throws IOException { + // Given — footer.fbs includes the other schemas and declares three root types. + Ast.SchemaFile sut = parse(repoRoot().resolve("core/src/main/flatbuffers/vortex-file/footer.fbs")); + + // Then + assertThat(sut.includes()).containsExactly("vortex-array/array.fbs", "vortex-layout/layout.fbs"); + assertThat(sut.rootTypes()).containsExactly("FileStatistics", "Footer", "Postscript"); + } + } + + @Nested + class Synthetic { + + @Test + void normalizesScalarAliasesToCanonicalWidths() { + // Given — width aliases that must collapse to the same canonical scalars. + String src = """ + namespace x; + table T { + a: byte; + b: int8; + c: ulong; + d: uint64; + } + """; + + // When + Ast.SchemaFile sut = new Parser(new Lexer(src).tokenize()).parseFile(); + + // Then — byte == int8 == INT8, ulong == uint64 == UINT64. + Ast.TableDecl t = (Ast.TableDecl) sut.decls().getFirst(); + assertThat(t.fields().get(0).type()).isEqualTo(Ast.Scalar.INT8); + assertThat(t.fields().get(1).type()).isEqualTo(Ast.Scalar.INT8); + assertThat(t.fields().get(2).type()).isEqualTo(Ast.Scalar.UINT64); + assertThat(t.fields().get(3).type()).isEqualTo(Ast.Scalar.UINT64); + } + + @Test + void honorsRequiredAttribute() { + // Given — a `(required)` string field. + String src = "namespace x; table T { id: string (required); }"; + + // When + Ast.SchemaFile sut = new Parser(new Lexer(src).tokenize()).parseFile(); + + // Then + Ast.TableDecl t = (Ast.TableDecl) sut.decls().getFirst(); + assertThat(t.fields().getFirst().required()).isTrue(); + } + + @Test + void rejectsUnterminatedTable() { + // Given — a table missing its closing brace runs into EOF. + String src = "namespace x; table T { a: int;"; + + // When + Then + assertThatThrownBy(() -> new Parser(new Lexer(src).tokenize()).parseFile()) + .isInstanceOf(FbsParseException.class); + } + } + + private static Ast.SchemaFile parse(Path p) throws IOException { + return new Parser(new Lexer(Files.readString(p)).tokenize()).parseFile(); + } + + private static Path repoRoot() { + return Path.of("").toAbsolutePath().getParent(); + } + + private static Ast.Decl declNamed(Ast.SchemaFile file, String name) { + return file.decls().stream() + .filter(d -> d.name().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("no decl named " + name)); + } + + private static Ast.Field fieldNamed(java.util.List fields, String name) { + return fields.stream() + .filter(f -> f.name().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("no field named " + name)); + } +} diff --git a/inspector/pom.xml b/inspector/pom.xml index f8978758a..b1ee96185 100644 --- a/inspector/pom.xml +++ b/inspector/pom.xml @@ -24,10 +24,6 @@ io.github.dfa1.vortex vortex-reader - - com.google.flatbuffers - flatbuffers-java - diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index d86fb408b..b37059684 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -1,17 +1,17 @@ package io.github.dfa1.vortex.inspect; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; + import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.reader.Footer; import io.github.dfa1.vortex.reader.Layout; import io.github.dfa1.vortex.reader.SegmentSpec; -import io.github.dfa1.vortex.fbs.Array; -import io.github.dfa1.vortex.fbs.ArrayNode; +import io.github.dfa1.vortex.fbs.FbsArray; +import io.github.dfa1.vortex.fbs.FbsArrayNode; import io.github.dfa1.vortex.reader.VortexHandle; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; @@ -255,12 +255,10 @@ public interface Progress { private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) { int segLen = (int) seg.byteSize(); - ByteBuffer bb = seg.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); - int fbLen = bb.getInt(segLen - 4); + int fbLen = seg.get(LE_INT, segLen - 4); int fbStart = segLen - 4 - fbLen; - ByteBuffer fbBuf = bb.slice(fbStart, fbLen).order(ByteOrder.LITTLE_ENDIAN); - Array fbArray = Array.getRootAsArray(fbBuf); - ArrayNode root = fbArray.root(); + FbsArray fbArray = FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); + FbsArrayNode root = fbArray.root(); if (root == null) { return new Peek(null, ArrayStats.empty()); } diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ZonedStatsSchema.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ZonedStatsSchema.java index e7ba1e0b1..8ec63cd20 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ZonedStatsSchema.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ZonedStatsSchema.java @@ -2,7 +2,10 @@ import io.github.dfa1.vortex.core.DType; -import java.nio.ByteBuffer; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import java.util.ArrayList; import java.util.List; @@ -79,12 +82,11 @@ private ZonedStatsSchema() { /// /// @param metadata raw `vortex.stats` layout metadata, possibly `null` /// @return decoded zone length, or `0` when absent - public static long zoneLength(ByteBuffer metadata) { - if (metadata == null || metadata.remaining() < 4) { + public static long zoneLength(MemorySegment metadata) { + if (metadata == null || metadata.byteSize() < 4) { return 0L; } - ByteBuffer mb = metadata.duplicate().order(java.nio.ByteOrder.LITTLE_ENDIAN); - return Integer.toUnsignedLong(mb.getInt(mb.position())); + return Integer.toUnsignedLong(metadata.get(LE_INT, 0)); } /// Returns the stats present in the layout metadata bitset, in ordinal order. @@ -95,19 +97,17 @@ public static long zoneLength(ByteBuffer metadata) { /// /// @param metadata raw `vortex.stats` layout metadata, possibly `null` /// @return present stats in ascending ordinal order; empty if metadata carries no bitset - public static List presentStats(ByteBuffer metadata) { - if (metadata == null || metadata.remaining() <= 4) { + public static List presentStats(MemorySegment metadata) { + if (metadata == null || metadata.byteSize() <= 4) { return List.of(); } - ByteBuffer mb = metadata.duplicate(); - mb.position(mb.position() + 4); Stat[] all = Stat.values(); List out = new ArrayList<>(all.length); int bitIndex = 0; - while (mb.hasRemaining()) { - byte b = mb.get(); + for (long pos = 4; pos < metadata.byteSize(); pos++) { + int b = metadata.get(ValueLayout.JAVA_BYTE, pos) & 0xff; for (int bit = 0; bit < 8; bit++) { - if (((b & 0xff) & (1 << bit)) != 0 && bitIndex < all.length) { + if ((b & (1 << bit)) != 0 && bitIndex < all.length) { out.add(all[bitIndex]); } bitIndex++; @@ -130,7 +130,7 @@ public static List presentStats(ByteBuffer metadata) { /// @param columnDtype the column's logical dtype (the `data` child's dtype) /// @param metadata raw `vortex.stats` layout metadata /// @return reconstructed non-nullable struct dtype, or empty struct when nothing is present - public static DType.Struct statsTableDtype(DType columnDtype, ByteBuffer metadata) { + public static DType.Struct statsTableDtype(DType columnDtype, MemorySegment metadata) { return statsTableDtype(columnDtype, presentStats(metadata)); } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ZonedStatsSchemaTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ZonedStatsSchemaTest.java index e377994eb..7742a47db 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ZonedStatsSchemaTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ZonedStatsSchemaTest.java @@ -1,12 +1,12 @@ package io.github.dfa1.vortex.inspect; +import java.lang.foreign.MemorySegment; + import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.PType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.EnumSet; import java.util.List; @@ -19,8 +19,8 @@ class ZoneLength { @Test void readsLittleEndianU32() { // Given — 8192 = 0x2000 stored as LE u32 - ByteBuffer meta = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); - meta.putInt(0, 8192); + MemorySegment meta = MemorySegment.ofArray(new byte[4]); + meta.set(io.github.dfa1.vortex.encoding.PTypeIO.LE_INT, 0, 8192); // When long result = ZonedStatsSchema.zoneLength(meta); @@ -41,7 +41,7 @@ void returnsZeroForNullMetadata() { @Test void returnsZeroForShortMetadata() { // Given — only 2 bytes (insufficient for a u32) - ByteBuffer meta = ByteBuffer.allocate(2); + MemorySegment meta = MemorySegment.ofArray(new byte[2]); // When long result = ZonedStatsSchema.zoneLength(meta); @@ -56,7 +56,7 @@ class PresentStats { @Test void decodesBitsetLsbFirst() { // Given — bits for Min (4) and Max (3) set: 0b0001_1000 = 0x18 - ByteBuffer meta = metaWithBitset(8192, 0x18); + MemorySegment meta = metaWithBitset(8192, 0x18); // When List stats = ZonedStatsSchema.presentStats(meta); @@ -70,10 +70,10 @@ void decodesMultiByteBitset() { // Given — set bits for MAX (3), MIN (4), NULL_COUNT (6), NAN_COUNT (8) // byte 0: 0b0101_1000 = 0x58 (bits 3,4,6) // byte 1: 0b0000_0001 = 0x01 (bit 8 → bit 0 of second byte) - ByteBuffer meta = ByteBuffer.allocate(4 + 2).order(ByteOrder.LITTLE_ENDIAN); - meta.putInt(0, 8192); - meta.put(4, (byte) 0x58); - meta.put(5, (byte) 0x01); + MemorySegment meta = MemorySegment.ofArray(new byte[4 + 2]); + meta.set(io.github.dfa1.vortex.encoding.PTypeIO.LE_INT, 0, 8192); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) 0x58); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 5, (byte) 0x01); // When List stats = ZonedStatsSchema.presentStats(meta); @@ -89,7 +89,7 @@ void decodesMultiByteBitset() { @Test void returnsEmptyWhenMetadataHasNoBitset() { // Given — exactly 4 bytes (zone length only, no bitset trailer) - ByteBuffer meta = ByteBuffer.allocate(4); + MemorySegment meta = MemorySegment.ofArray(new byte[4]); // When List stats = ZonedStatsSchema.presentStats(meta); @@ -102,10 +102,10 @@ void returnsEmptyWhenMetadataHasNoBitset() { void ignoresFutureStatBits() { // Given — bit 31 set (beyond any known stat) plus MAX/MIN // byte 0: 0x18 (MAX|MIN), bytes 1-2: 0, byte 3: 0x80 (bit 31) - ByteBuffer meta = ByteBuffer.allocate(4 + 4).order(ByteOrder.LITTLE_ENDIAN); - meta.putInt(0, 8192); - meta.put(4, (byte) 0x18); - meta.put(7, (byte) 0x80); + MemorySegment meta = MemorySegment.ofArray(new byte[4 + 4]); + meta.set(io.github.dfa1.vortex.encoding.PTypeIO.LE_INT, 0, 8192); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) 0x18); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 7, (byte) 0x80); // When List stats = ZonedStatsSchema.presentStats(meta); @@ -244,10 +244,10 @@ void allStatsTogetherForI32() { } } - private static ByteBuffer metaWithBitset(int zoneLen, int firstByte) { - ByteBuffer meta = ByteBuffer.allocate(4 + 1).order(ByteOrder.LITTLE_ENDIAN); - meta.putInt(0, zoneLen); - meta.put(4, (byte) firstByte); + private static MemorySegment metaWithBitset(int zoneLen, int firstByte) { + MemorySegment meta = MemorySegment.ofArray(new byte[4 + 1]); + meta.set(io.github.dfa1.vortex.encoding.PTypeIO.LE_INT, 0, zoneLen); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) firstByte); return meta; } } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/PcoFixtureInspectionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/PcoFixtureInspectionIntegrationTest.java index 9ebafb809..7b6ef132a 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/PcoFixtureInspectionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/PcoFixtureInspectionIntegrationTest.java @@ -4,9 +4,9 @@ import io.github.dfa1.vortex.reader.Layout; import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.reader.ReadRegistry; -import io.github.dfa1.vortex.fbs.Array; -import io.github.dfa1.vortex.fbs.ArrayNode; -import io.github.dfa1.vortex.fbs.Buffer; +import io.github.dfa1.vortex.fbs.FbsArray; +import io.github.dfa1.vortex.fbs.FbsArrayNode; +import io.github.dfa1.vortex.fbs.FbsBuffer; import io.github.dfa1.vortex.reader.VortexReader; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -30,7 +30,7 @@ /// Phase 0 scoping for `vortex.pco` decode port. /// /// Inspects each pco-blocked fixture: walks layout, locates `vortex.pco`-encoded -/// `ArrayNode`s, hand-parses the `PcoMetadata` proto (no generated stub needed yet), +/// `FbsArrayNode`s, hand-parses the `PcoMetadata` proto (no generated stub needed yet), /// and dumps mode/delta nibbles from the first chunk_meta buffer. /// /// Output drives the Phase 5 scope decision (which pco modes / delta encodings to @@ -118,7 +118,7 @@ private static void scanFlatSegment(MemorySegment seg, List arraySpecs, int fbLen = bb.getInt(segLen - 4); int fbStart = segLen - 4 - fbLen; ByteBuffer fbBuf = bb.slice(fbStart, fbLen).order(ByteOrder.LITTLE_ENDIAN); - Array fbArray = Array.getRootAsArray(fbBuf); + FbsArray fbArray = FbsArray.getRootAsFbsArray(java.lang.foreign.MemorySegment.ofBuffer(fbBuf)); // Materialize per-buffer offsets (need to read chunk_meta buffer for pco arrays). int numBuffers = fbArray.buffersLength(); @@ -126,7 +126,7 @@ private static void scanFlatSegment(MemorySegment seg, List arraySpecs, long[] bufLengths = new long[numBuffers]; long dataOffset = 0; for (int i = 0; i < numBuffers; i++) { - Buffer bufDesc = fbArray.buffers(i); + FbsBuffer bufDesc = fbArray.buffers(i); dataOffset += bufDesc.padding(); bufOffsets[i] = dataOffset; bufLengths[i] = bufDesc.length(); @@ -136,7 +136,7 @@ private static void scanFlatSegment(MemorySegment seg, List arraySpecs, visitNode(fbArray.root(), arraySpecs, seg, bufOffsets, bufLengths, stats, columnPath); } - private static void visitNode(ArrayNode node, List arraySpecs, MemorySegment seg, + private static void visitNode(FbsArrayNode node, List arraySpecs, MemorySegment seg, long[] bufOffsets, long[] bufLengths, PcoStats stats, String columnPath) { if (node == null) { return; @@ -150,14 +150,14 @@ private static void visitNode(ArrayNode node, List arraySpecs, MemorySeg } } - private static void recordPco(ArrayNode node, MemorySegment seg, + private static void recordPco(FbsArrayNode node, MemorySegment seg, long[] bufOffsets, long[] bufLengths, PcoStats stats, String columnPath) { stats.pcoArrayCount++; stats.columnPaths.add(columnPath); - ByteBuffer meta = node.metadataAsByteBuffer(); - ByteBuffer metaSlice = meta != null ? meta.slice() : null; + java.lang.foreign.MemorySegment metaSeg = node.metadataAsSegment(); + ByteBuffer metaSlice = metaSeg != null ? metaSeg.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN) : null; PcoMeta pm = metaSlice != null ? parsePcoMetadata(metaSlice) : new PcoMeta(); stats.chunksPerArray.add(pm.chunkCount); stats.pagesPerArray.add(pm.pageCount); diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VariantJavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VariantJavaWritesRustReadsIntegrationTest.java index 38df8bc3d..a6cfcb5f5 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/VariantJavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VariantJavaWritesRustReadsIntegrationTest.java @@ -5,9 +5,9 @@ import dev.vortex.arrow.ArrowAllocation; import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.DType; -import io.github.dfa1.vortex.proto.Primitive; -import io.github.dfa1.vortex.proto.Scalar; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoPrimitive; +import io.github.dfa1.vortex.proto.ProtoScalar; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.writer.encode.VariantData; @@ -48,7 +48,7 @@ class VariantJavaWritesRustReadsIntegrationTest { @Test void javaWriter_jniReader_constantVariantColumn(@TempDir Path tmp) throws IOException { // Given — a constant variant column: every one of N rows is the i32 variant value 7. - // Built as Rust's Scalar::variant(Scalar::primitive(7i32)): the inner Scalar carries + // Built as Rust's ProtoScalar::variant(ProtoScalar::primitive(7i32)): the inner ProtoScalar carries // its own i32 dtype so the reference reader knows the wrapped value's type. Path file = tmp.resolve("java_variant.vtx"); int rows = 5; @@ -75,7 +75,7 @@ void javaWriter_jniReader_varyingVariantColumn(@TempDir Path tmp) throws IOExcep // lays this out as core_storage = vortex.chunked of one vortex.constant per row, // exactly the representation the Rust reference uses for a row-varying variant array. Path file = tmp.resolve("java_variant_varying.vtx"); - List values = List.of(i32Variant(10L), i32Variant(20L), i32Variant(30L), i32Variant(40L)); + List values = List.of(i32Variant(10L), i32Variant(20L), i32Variant(30L), i32Variant(40L)); VariantData data = new VariantData(values); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); @@ -95,7 +95,7 @@ void javaWriter_jniReader_shreddedVariantColumn(@TempDir Path tmp) throws IOExce // Given — a variant column with a row-aligned shredded i32 projection. The container // gains a second (shredded) typed child plus shredded_dtype in its metadata. Path file = tmp.resolve("java_variant_shredded.vtx"); - List values = List.of(i32Variant(10L), i32Variant(20L), i32Variant(30L)); + List values = List.of(i32Variant(10L), i32Variant(20L), i32Variant(30L)); VariantData data = VariantData.shredded( values, new int[]{10, 20, 30}, DType.I32); @@ -111,12 +111,12 @@ void javaWriter_jniReader_shreddedVariantColumn(@TempDir Path tmp) throws IOExce assertThat(ds.arrowSchema(ALLOCATOR).getFields()).extracting(Field::getName).contains("v"); } - private static Scalar i32Variant(long value) { + private static ProtoScalar i32Variant(long value) { // Inner typed scalar carrying its own i32 dtype, wrapped as a variant value - // (mirrors Rust Scalar::variant(Scalar::primitive(value))). - return new Scalar( - io.github.dfa1.vortex.proto.DType.ofPrimitive( - new Primitive(io.github.dfa1.vortex.proto.PType.I32, false)), - ScalarValue.ofInt64Value(value)); + // (mirrors Rust ProtoScalar::variant(ProtoScalar::primitive(value))). + return new ProtoScalar( + io.github.dfa1.vortex.proto.ProtoDType.ofPrimitive( + new ProtoPrimitive(io.github.dfa1.vortex.proto.ProtoPType.I32, false)), + ProtoScalarValue.ofInt64Value(value)); } } diff --git a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetImporter.java b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetImporter.java index 57de76a89..52f8c3f7a 100644 --- a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetImporter.java +++ b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetImporter.java @@ -12,11 +12,10 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.writer.encode.DateTimePartsData; import io.github.dfa1.vortex.encoding.TimeUnit; +import io.github.dfa1.vortex.extension.TimestampDtype; import io.github.dfa1.vortex.writer.VortexWriter; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -157,12 +156,8 @@ private static DType.Extension timestampExtension(LogicalType.TimeUnit parquetUn case MICROS -> TimeUnit.Microseconds; case NANOS -> TimeUnit.Nanoseconds; }; - // Extension metadata: byte[0]=unit tag, bytes[1-2]=tz_len u16 LE (0 = no tz) - ByteBuffer meta = ByteBuffer.allocate(3).order(ByteOrder.LITTLE_ENDIAN); - meta.put((byte) unit.ordinal()); - meta.putShort((short) 0); - meta.flip(); - return new DType.Extension("vortex.timestamp", new DType.Primitive(PType.I64, nullable), meta, nullable); + // The wire layout (TimeUnit tag + tz_len + tz UTF-8) lives in TimestampDtype — reuse it. + return TimestampDtype.of(unit, null, nullable); } private static DType mapByteArray(LogicalType logical, boolean nullable, String colName) { diff --git a/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java b/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java index bc66e8426..6e0bc8931 100644 --- a/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java +++ b/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java @@ -1,16 +1,18 @@ package io.github.dfa1.vortex.performance; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; + import dev.vortex.api.Session; import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.fbs.Array; -import io.github.dfa1.vortex.fbs.ArrayNode; -import io.github.dfa1.vortex.fbs.Buffer; +import io.github.dfa1.vortex.fbs.FbsArray; +import io.github.dfa1.vortex.fbs.FbsArrayNode; +import io.github.dfa1.vortex.fbs.FbsBuffer; import io.github.dfa1.vortex.parquet.ParquetImporter; -import io.github.dfa1.vortex.proto.ALPMetadata; -import io.github.dfa1.vortex.proto.BitPackedMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPMetadata; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.Footer; import io.github.dfa1.vortex.reader.Layout; import io.github.dfa1.vortex.reader.ReadRegistry; @@ -25,7 +27,7 @@ import java.nio.file.Path; import java.util.List; -/// Per-column encoding-tree diff: prints the full ArrayNode chain (encoding id + +/// Per-column encoding-tree diff: prints the full FbsArrayNode chain (encoding id + /// decoded metadata + buffer byte counts) for a single column in the Rust JNI /// file and the Java file side-by-side. Use to investigate why a column is /// larger in Java by comparing the encoding choices chunk-by-chunk. @@ -122,15 +124,13 @@ private static void walkLayout(VortexReader reader, Layout layout, Footer footer private static void dumpFlatRoot(MemorySegment seg, List arraySpecs, String indent) { int segLen = (int) seg.byteSize(); - ByteBuffer bb = seg.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); - int fbLen = bb.getInt(segLen - 4); + int fbLen = seg.get(LE_INT, segLen - 4); int fbStart = segLen - 4 - fbLen; - ByteBuffer fbBuf = bb.slice(fbStart, fbLen).order(ByteOrder.LITTLE_ENDIAN); - Array arr = Array.getRootAsArray(fbBuf); + FbsArray arr = FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); int bufCount = arr.buffersLength(); long[] bufBytes = new long[bufCount]; for (int i = 0; i < bufCount; i++) { - Buffer b = arr.buffers(i); + FbsBuffer b = arr.buffers(i); bufBytes[i] = b.length(); } long dataBytes = (long) segLen - 4L - fbLen; @@ -141,7 +141,7 @@ private static void dumpFlatRoot(MemorySegment seg, List arraySpecs, Str dumpNode(arr.root(), arraySpecs, bufBytes, indent + " "); } - private static void dumpNode(ArrayNode node, List arraySpecs, long[] bufBytes, String indent) { + private static void dumpNode(FbsArrayNode node, List arraySpecs, long[] bufBytes, String indent) { if (node == null) { return; } @@ -158,7 +158,8 @@ private static void dumpNode(ArrayNode node, List arraySpecs, long[] buf line.append(" buf=").append(java.util.Arrays.toString(bufs)); line.append(" bytes=").append(String.format("%,d", nodeBytes)); } - ByteBuffer meta = node.metadataAsByteBuffer(); + java.lang.foreign.MemorySegment metaSeg = node.metadataAsSegment(); + ByteBuffer meta = metaSeg != null ? metaSeg.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN) : null; if (meta != null && meta.remaining() > 0) { line.append(" ").append(decodeMeta(id, meta)); } @@ -172,12 +173,12 @@ private static String decodeMeta(String id, ByteBuffer meta) { try { MemorySegment seg = MemorySegment.ofBuffer(meta.duplicate()); if (EncodingId.FASTLANES_BITPACKED.id().equals(id)) { - BitPackedMetadata m = BitPackedMetadata.decode(seg, 0, seg.byteSize()); + ProtoBitPackedMetadata m = ProtoBitPackedMetadata.decode(seg, 0, seg.byteSize()); return "bitWidth=" + m.bit_width() + " offset=" + m.offset() + " patches=" + describePatches(m.patches()); } if (EncodingId.VORTEX_ALP.id().equals(id)) { - ALPMetadata m = ALPMetadata.decode(seg, 0, seg.byteSize()); + ProtoALPMetadata m = ProtoALPMetadata.decode(seg, 0, seg.byteSize()); return "expE=" + m.exp_e() + " expF=" + m.exp_f() + " patches=" + describePatches(m.patches()); } @@ -187,7 +188,7 @@ private static String decodeMeta(String id, ByteBuffer meta) { } } - private static String describePatches(PatchesMetadata p) { + private static String describePatches(ProtoPatchesMetadata p) { if (p == null) { return "none"; } diff --git a/pom.xml b/pom.xml index e4637e1ab..6f2932302 100644 --- a/pom.xml +++ b/pom.xml @@ -39,6 +39,7 @@ proto-gen + fbs-gen core reader writer @@ -58,7 +59,6 @@ 3.6 - 25.2.10 4.3.0 1.0.0.CR2 1.37 @@ -190,11 +190,6 @@ ${zstd-jni.version} - - com.google.flatbuffers - flatbuffers-java - ${flatbuffers.version} - org.openjdk.jmh jmh-core diff --git a/proto-gen/src/main/java/io/github/dfa1/vortex/protogen/TypeRegistry.java b/proto-gen/src/main/java/io/github/dfa1/vortex/protogen/TypeRegistry.java index 13c3e8766..8a93607cb 100644 --- a/proto-gen/src/main/java/io/github/dfa1/vortex/protogen/TypeRegistry.java +++ b/proto-gen/src/main/java/io/github/dfa1/vortex/protogen/TypeRegistry.java @@ -122,12 +122,26 @@ public Iterable all() { /// A resolved type — either a message or an enum — carrying enough context for code emission. public sealed interface ResolvedType { + /// Prefix applied to every emitted Java class name so that the generated proto + /// records (`DType`, `Bool`, `List`, …) do not collide with project or JDK types + /// of the same name on the consumer classpath. + String JAVA_NAME_PREFIX = "Proto"; + /// @return Java package for the emitted class String javaPackage(); /// @return Java simple class name String javaName(); + /// Extracts the last dot-separated segment of a fully-qualified proto name. + /// + /// @param fqn fully-qualified proto name + /// @return the trailing segment (the proto type's own name) + private static String simpleName(String fqn) { + int dot = fqn.lastIndexOf('.'); + return dot >= 0 ? fqn.substring(dot + 1) : fqn; + } + /// A resolved message type. /// /// @param decl AST node @@ -136,8 +150,7 @@ public sealed interface ResolvedType { record Message(Ast.MessageDecl decl, String fqn, String javaPackage) implements ResolvedType { @Override public String javaName() { - int dot = fqn.lastIndexOf('.'); - return dot >= 0 ? fqn.substring(dot + 1) : fqn; + return JAVA_NAME_PREFIX + simpleName(fqn); } } @@ -149,8 +162,7 @@ public String javaName() { record Enum(Ast.EnumDecl decl, String fqn, String javaPackage) implements ResolvedType { @Override public String javaName() { - int dot = fqn.lastIndexOf('.'); - return dot >= 0 ? fqn.substring(dot + 1) : fqn; + return JAVA_NAME_PREFIX + simpleName(fqn); } } } diff --git a/proto-gen/src/test/java/io/github/dfa1/vortex/protogen/CodeGenTest.java b/proto-gen/src/test/java/io/github/dfa1/vortex/protogen/CodeGenTest.java index 2939f1a0f..41b3b2d2c 100644 --- a/proto-gen/src/test/java/io/github/dfa1/vortex/protogen/CodeGenTest.java +++ b/proto-gen/src/test/java/io/github/dfa1/vortex/protogen/CodeGenTest.java @@ -27,11 +27,11 @@ void generatesJavaFromAllVortexProtos(@TempDir Path tmp) throws IOException { sut.emit(tmp); // Then — a few expected files were emitted. - assertThat(Files.exists(tmp.resolve("PType.java"))).isTrue(); - assertThat(Files.exists(tmp.resolve("DType.java"))).isTrue(); - assertThat(Files.exists(tmp.resolve("ScalarValue.java"))).isTrue(); - assertThat(Files.exists(tmp.resolve("BitPackedMetadata.java"))).isTrue(); - assertThat(Files.exists(tmp.resolve("NullValue.java"))).isTrue(); + assertThat(Files.exists(tmp.resolve("ProtoPType.java"))).isTrue(); + assertThat(Files.exists(tmp.resolve("ProtoDType.java"))).isTrue(); + assertThat(Files.exists(tmp.resolve("ProtoScalarValue.java"))).isTrue(); + assertThat(Files.exists(tmp.resolve("ProtoBitPackedMetadata.java"))).isTrue(); + assertThat(Files.exists(tmp.resolve("ProtoNullValue.java"))).isTrue(); } @Test @@ -46,14 +46,14 @@ void generatedBitPackedMetadataHasExpectedComponents(@TempDir Path tmp) throws I new CodeGen(new TypeRegistry(files)).emit(tmp); // When - String src = Files.readString(tmp.resolve("BitPackedMetadata.java")); + String src = Files.readString(tmp.resolve("ProtoBitPackedMetadata.java")); // Then - assertThat(src).contains("public record BitPackedMetadata("); + assertThat(src).contains("public record ProtoBitPackedMetadata("); assertThat(src).contains("int bit_width"); assertThat(src).contains("int offset"); - assertThat(src).contains("PatchesMetadata patches"); - assertThat(src).contains("public static BitPackedMetadata decode(MemorySegment __seg, long __off, long __len)"); + assertThat(src).contains("ProtoPatchesMetadata patches"); + assertThat(src).contains("public static ProtoBitPackedMetadata decode(MemorySegment __seg, long __off, long __len)"); assertThat(src).contains("public byte[] encode()"); } diff --git a/reader/pom.xml b/reader/pom.xml index a0f06d329..dc5e19051 100644 --- a/reader/pom.xml +++ b/reader/pom.xml @@ -20,10 +20,6 @@ io.github.dfa1.vortex vortex-core - - com.google.flatbuffers - flatbuffers-java - io.airlift aircompressor-v3 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java index 12350f0bc..53b3ad792 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java @@ -1,11 +1,10 @@ package io.github.dfa1.vortex.reader; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; /// Per-array statistics embedded in the encoding tree. @@ -33,17 +32,17 @@ public static ArrayStats empty() { return EMPTY; } - /// Parses stats from a FlatBuffers [io.github.dfa1.vortex.fbs.ArrayStats] table. + /// Parses stats from a FlatBuffers [io.github.dfa1.vortex.fbs.FbsArrayStats] table. /// Returns an empty instance when `fbs` is `null` or carries no min/max and no null count. /// /// @param fbs the FlatBuffers stats table, or `null` /// @return parsed stats, or an empty instance if no usable data is present - public static ArrayStats fromFbs(io.github.dfa1.vortex.fbs.ArrayStats fbs) { + public static ArrayStats fromFbs(io.github.dfa1.vortex.fbs.FbsArrayStats fbs) { if (fbs == null) { return EMPTY; } - Object min = decodeScalar(fbs.minAsByteBuffer()); - Object max = decodeScalar(fbs.maxAsByteBuffer()); + Object min = decodeScalar(fbs.minAsSegment()); + Object max = decodeScalar(fbs.maxAsSegment()); Long nullCount = fbs.hasNullCount() ? fbs.nullCount() : null; if (min == null && max == null && nullCount == null) { return EMPTY; @@ -51,13 +50,12 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.fbs.ArrayStats fbs) { return new ArrayStats(min, max, null, nullCount, null, null); } - private static Object decodeScalar(ByteBuffer bytes) { - if (bytes == null || !bytes.hasRemaining()) { + private static Object decodeScalar(MemorySegment seg) { + if (seg == null || seg.byteSize() == 0) { return null; } try { - MemorySegment seg = MemorySegment.ofBuffer(bytes.duplicate()); - ScalarValue sv = ScalarValue.decode(seg, 0, seg.byteSize()); + ProtoScalarValue sv = ProtoScalarValue.decode(seg, 0, seg.byteSize()); if (sv.int64_value() != null) { return sv.int64_value(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java index ee28272be..e5b2cabcf 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java @@ -1,10 +1,12 @@ package io.github.dfa1.vortex.reader; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; + import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.IoBounds; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.fbs.Buffer; +import io.github.dfa1.vortex.fbs.FbsBuffer; import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.reader.decode.KnownArrayNode; @@ -12,8 +14,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.List; /// Parses a flat segment from the memory-mapped file region and dispatches to the @@ -46,21 +46,19 @@ public FlatSegmentDecoder(ReadRegistry registry) { public Array decode(MemorySegment seg, List encodingSpecs, DType dtype, long rowCount, SegmentAllocator arena) { int segLen = IoBounds.toIntSize(seg.byteSize()); - ByteBuffer bb = seg.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); // The trailing u32 length field must itself be in range before we read it. IoBounds.checkRange(segLen - 4L, 4, segLen); - int fbLen = bb.getInt(segLen - 4); + int fbLen = seg.get(LE_INT, segLen - 4); long fbStart = segLen - 4L - fbLen; IoBounds.checkRange(fbStart, fbLen, segLen); - ByteBuffer fbBuf = bb.slice((int) fbStart, fbLen).order(ByteOrder.LITTLE_ENDIAN); - var fbArray = io.github.dfa1.vortex.fbs.Array.getRootAsArray(fbBuf); + var fbArray = io.github.dfa1.vortex.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); int numBuffers = IoBounds.checkCount(fbArray.buffersLength()); MemorySegment[] bufs = new MemorySegment[numBuffers]; long dataOffset = 0; for (int i = 0; i < numBuffers; i++) { - Buffer bufDesc = fbArray.buffers(i); + FbsBuffer bufDesc = fbArray.buffers(i); dataOffset += bufDesc.padding(); bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); dataOffset += bufDesc.length(); @@ -72,7 +70,7 @@ public Array decode(MemorySegment seg, List encodingSpecs, } private static ArrayNode convertArrayNode( - io.github.dfa1.vortex.fbs.ArrayNode fbs, + io.github.dfa1.vortex.fbs.FbsArrayNode fbs, List encodingSpecs ) { String rawEncodingId = encodingSpecs.get(fbs.encoding()); @@ -87,8 +85,7 @@ private static ArrayNode convertArrayNode( bufferIndices[i] = fbs.buffers(i); } - ByteBuffer rawMeta = fbs.metadataAsByteBuffer(); - ByteBuffer meta = (rawMeta != null) ? rawMeta.slice() : null; + MemorySegment meta = fbs.metadataAsSegment(); return EncodingId.parse(rawEncodingId) .map(known -> new KnownArrayNode(known, meta, children, bufferIndices)) .orElseGet(() -> new UnknownArrayNode(rawEncodingId, meta, children, bufferIndices)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Layout.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Layout.java index d87fead3e..08d80b9b3 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Layout.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Layout.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.util.List; /// Node in the Vortex layout tree. Describes physical arrangement of data in the file. @@ -18,7 +18,7 @@ public record Layout( String encodingId, long rowCount, - ByteBuffer metadata, + MemorySegment metadata, List children, List segments ) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index 40c53a300..991178984 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -4,21 +4,19 @@ import io.github.dfa1.vortex.core.IoBounds; import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.fbs.Binary; -import io.github.dfa1.vortex.fbs.Bool; -import io.github.dfa1.vortex.fbs.Decimal; -import io.github.dfa1.vortex.fbs.Extension; -import io.github.dfa1.vortex.fbs.FixedSizeList; -import io.github.dfa1.vortex.fbs.Postscript; -import io.github.dfa1.vortex.fbs.Primitive; -import io.github.dfa1.vortex.fbs.Struct_; -import io.github.dfa1.vortex.fbs.Type; -import io.github.dfa1.vortex.fbs.Utf8; -import io.github.dfa1.vortex.fbs.Variant; +import io.github.dfa1.vortex.fbs.FbsBinary; +import io.github.dfa1.vortex.fbs.FbsBool; +import io.github.dfa1.vortex.fbs.FbsDecimal; +import io.github.dfa1.vortex.fbs.FbsExtension; +import io.github.dfa1.vortex.fbs.FbsFixedSizeList; +import io.github.dfa1.vortex.fbs.FbsPostscript; +import io.github.dfa1.vortex.fbs.FbsPrimitive; +import io.github.dfa1.vortex.fbs.FbsStruct_; +import io.github.dfa1.vortex.fbs.FbsType; +import io.github.dfa1.vortex.fbs.FbsUtf8; +import io.github.dfa1.vortex.fbs.FbsVariant; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; @@ -27,11 +25,11 @@ final class PostscriptParser { /// Hard cap on layout-tree recursion depth. Real-world layouts are typically four levels /// (Struct → Zoned → Chunked → Flat); 64 is well past any expected schema and prevents /// adversarial inputs — deeply nested trees or self-referential FlatBuffer cycles — from - /// blowing the JVM stack during [#convertLayout(io.github.dfa1.vortex.fbs.Layout, List, int)]. + /// blowing the JVM stack during [#convertLayout(io.github.dfa1.vortex.fbs.FbsLayout, List, int)]. static final int MAX_LAYOUT_DEPTH = 64; /// Hard cap on per-layout metadata size. The FlatBuffer runtime returns an unbounded slice - /// from `metadataAsByteBuffer()`; a crafted file can claim a multi-gigabyte metadata + /// from `metadataAsSegment()`; a crafted file can claim a multi-gigabyte metadata /// blob and force later allocators into pathological behaviour. 4 MiB is well above any /// real encoding's metadata footprint (the largest is FSST's symbol table at ~32 KiB). static final int MAX_LAYOUT_METADATA_BYTES = 4 * 1024 * 1024; @@ -39,8 +37,8 @@ final class PostscriptParser { private PostscriptParser() { } - static ParsedFile parse(ByteBuffer postscriptBuf, MemorySegment fileSegment, long fileSize) { - var ps = Postscript.getRootAsPostscript(postscriptBuf); + static ParsedFile parse(MemorySegment postscriptSeg, MemorySegment fileSegment, long fileSize) { + var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); var footerSeg = ps.footer(); if (footerSeg == null) { @@ -58,9 +56,9 @@ static ParsedFile parse(ByteBuffer postscriptBuf, MemorySegment fileSegment, lon checkBlobBounds("dtype", dtypeSeg.offset(), dtypeSeg.length(), fileSize); } - ByteBuffer footerBuf = slice(fileSegment, footerSeg.offset(), footerSeg.length()); - ByteBuffer layoutBuf = slice(fileSegment, layoutSeg.offset(), layoutSeg.length()); - ByteBuffer dtypeBuf = (dtypeSeg != null && dtypeSeg.length() > 0) + MemorySegment footerBuf = slice(fileSegment, footerSeg.offset(), footerSeg.length()); + MemorySegment layoutBuf = slice(fileSegment, layoutSeg.offset(), layoutSeg.length()); + MemorySegment dtypeBuf = (dtypeSeg != null && dtypeSeg.length() > 0) ? slice(fileSegment, dtypeSeg.offset(), dtypeSeg.length()) : null; @@ -102,17 +100,17 @@ private static void checkBlobBounds(String name, long offset, long length, long } } - static ParsedFile parseBlobs(ByteBuffer footerBuf, ByteBuffer layoutBuf, ByteBuffer dtypeBuf) { + static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, MemorySegment dtypeBuf) { try { - var fbsFooter = io.github.dfa1.vortex.fbs.Footer.getRootAsFooter(footerBuf); - var fbsLayout = io.github.dfa1.vortex.fbs.Layout.getRootAsLayout(layoutBuf); + var fbsFooter = io.github.dfa1.vortex.fbs.FbsFooter.getRootAsFbsFooter(footerBuf); + var fbsLayout = io.github.dfa1.vortex.fbs.FbsLayout.getRootAsFbsLayout(layoutBuf); Footer footer = convertFooter(fbsFooter); Layout layout = convertLayout(fbsLayout, footer.layoutSpecs(), 0); DType dtype = null; - if (dtypeBuf != null && dtypeBuf.hasRemaining()) { - dtype = convertDType(io.github.dfa1.vortex.fbs.DType.getRootAsDType(dtypeBuf)); + if (dtypeBuf != null && dtypeBuf.byteSize() > 0) { + dtype = convertDType(io.github.dfa1.vortex.fbs.FbsDType.getRootAsFbsDType(dtypeBuf)); } return new ParsedFile(footer, dtype, layout); @@ -123,11 +121,11 @@ static ParsedFile parseBlobs(ByteBuffer footerBuf, ByteBuffer layoutBuf, ByteBuf } } - private static ByteBuffer slice(MemorySegment seg, long offset, long length) { - return IoBounds.slice(seg, offset, length).asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + private static MemorySegment slice(MemorySegment seg, long offset, long length) { + return IoBounds.slice(seg, offset, length); } - static Footer convertFooter(io.github.dfa1.vortex.fbs.Footer f) { + static Footer convertFooter(io.github.dfa1.vortex.fbs.FbsFooter f) { var arraySpecs = new ArrayList(f.arraySpecsLength()); for (int i = 0; i < f.arraySpecsLength(); i++) { arraySpecs.add(f.arraySpecs(i).id()); @@ -144,7 +142,7 @@ static Footer convertFooter(io.github.dfa1.vortex.fbs.Footer f) { segmentSpecs.add(new SegmentSpec( s.offset(), s.length(), (byte) s.alignmentExponent(), - CompressionScheme.of(s._Compression()))); + CompressionScheme.of(s.compression()))); } var compressionSpecs = new ArrayList(f.compressionSpecsLength()); @@ -157,7 +155,7 @@ static Footer convertFooter(io.github.dfa1.vortex.fbs.Footer f) { List.copyOf(segmentSpecs), List.copyOf(compressionSpecs)); } - private static Layout convertLayout(io.github.dfa1.vortex.fbs.Layout l, List layoutSpecs, int depth) { + private static Layout convertLayout(io.github.dfa1.vortex.fbs.FbsLayout l, List layoutSpecs, int depth) { if (depth > MAX_LAYOUT_DEPTH) { throw new VortexException( "layout tree depth exceeds limit (" + MAX_LAYOUT_DEPTH + ")"); @@ -170,10 +168,10 @@ private static Layout convertLayout(io.github.dfa1.vortex.fbs.Layout l, List MAX_LAYOUT_METADATA_BYTES) { + MemorySegment metadata = l.metadataAsSegment(); + if (metadata != null && metadata.byteSize() > MAX_LAYOUT_METADATA_BYTES) { throw new VortexException( - "layout metadata size " + metadata.remaining() + "layout metadata size " + metadata.byteSize() + " exceeds limit (" + MAX_LAYOUT_METADATA_BYTES + ")"); } @@ -190,17 +188,17 @@ private static Layout convertLayout(io.github.dfa1.vortex.fbs.Layout l, List new DType.Null(true); - case Type.Bool -> new DType.Bool(((Bool) fbs.type(new Bool())).nullable()); - case Type.Primitive -> { - var p = (Primitive) fbs.type(new Primitive()); + case FbsType.FbsNull -> new DType.Null(true); + case FbsType.FbsBool -> new DType.Bool(fbs.type(new FbsBool()).nullable()); + case FbsType.FbsPrimitive -> { + var p = fbs.type(new FbsPrimitive()); yield new DType.Primitive(convertPType(p.ptype()), p.nullable()); } - case Type.Decimal -> { - var d = (Decimal) fbs.type(new Decimal()); + case FbsType.FbsDecimal -> { + var d = fbs.type(new FbsDecimal()); int precision = d.precision(); int scale = d.scale(); // IEEE 754-2008 decimal128 covers precision up to 38 digits; scale must be in @@ -216,59 +214,55 @@ private static DType convertDType(io.github.dfa1.vortex.fbs.DType fbs) { } yield new DType.Decimal((byte) precision, (byte) scale, d.nullable()); } - case Type.Utf8 -> new DType.Utf8(((Utf8) fbs.type(new Utf8())).nullable()); - case Type.Binary -> new DType.Binary(((Binary) fbs.type(new Binary())).nullable()); - case Type.Struct_ -> { - var s = (Struct_) fbs.type(new Struct_()); + case FbsType.FbsUtf8 -> new DType.Utf8(fbs.type(new FbsUtf8()).nullable()); + case FbsType.FbsBinary -> new DType.Binary(fbs.type(new FbsBinary()).nullable()); + case FbsType.FbsStruct_ -> { + var s = fbs.type(new FbsStruct_()); var names = new ArrayList(s.namesLength()); var types = new ArrayList(s.dtypesLength()); for (int i = 0; i < s.namesLength(); i++) { names.add(s.names(i)); } for (int i = 0; i < s.dtypesLength(); i++) { - types.add(convertDType(s.dtypes(new io.github.dfa1.vortex.fbs.DType(), i))); + types.add(convertDType(s.dtypes(i))); } yield new DType.Struct(List.copyOf(names), List.copyOf(types), s.nullable()); } - case Type.List -> { - var l = (io.github.dfa1.vortex.fbs.List) fbs.type(new io.github.dfa1.vortex.fbs.List()); - yield new DType.List( - convertDType(l.elementType(new io.github.dfa1.vortex.fbs.DType())), - l.nullable()); + case FbsType.FbsList -> { + var l = fbs.type(new io.github.dfa1.vortex.fbs.FbsList()); + yield new DType.List(convertDType(l.elementType()), l.nullable()); } - case Type.FixedSizeList -> { - var fsl = (FixedSizeList) fbs.type(new FixedSizeList()); - yield new DType.FixedSizeList( - convertDType(fsl.elementType(new io.github.dfa1.vortex.fbs.DType())), - (int) fsl.size(), fsl.nullable()); + case FbsType.FbsFixedSizeList -> { + var fsl = fbs.type(new FbsFixedSizeList()); + yield new DType.FixedSizeList(convertDType(fsl.elementType()), (int) fsl.size(), fsl.nullable()); } - case Type.Extension -> { - var e = (Extension) fbs.type(new Extension()); - DType storage = convertDType(e.storageDtype(new io.github.dfa1.vortex.fbs.DType())); + case FbsType.FbsExtension -> { + var e = fbs.type(new FbsExtension()); + DType storage = convertDType(e.storageDtype()); yield new DType.Extension( e.id(), storage, - e.metadataAsByteBuffer(), + e.metadataAsSegment(), storage.nullable()); } - case Type.Variant -> new DType.Variant(((Variant) fbs.type(new Variant())).nullable()); + case FbsType.FbsVariant -> new DType.Variant(fbs.type(new FbsVariant()).nullable()); default -> throw new VortexException("unsupported DType typeType=" + typeType); }; } private static PType convertPType(int fbsPType) { return switch (fbsPType) { - case io.github.dfa1.vortex.fbs.PType.U8 -> PType.U8; - case io.github.dfa1.vortex.fbs.PType.U16 -> PType.U16; - case io.github.dfa1.vortex.fbs.PType.U32 -> PType.U32; - case io.github.dfa1.vortex.fbs.PType.U64 -> PType.U64; - case io.github.dfa1.vortex.fbs.PType.I8 -> PType.I8; - case io.github.dfa1.vortex.fbs.PType.I16 -> PType.I16; - case io.github.dfa1.vortex.fbs.PType.I32 -> PType.I32; - case io.github.dfa1.vortex.fbs.PType.I64 -> PType.I64; - case io.github.dfa1.vortex.fbs.PType.F16 -> PType.F16; - case io.github.dfa1.vortex.fbs.PType.F32 -> PType.F32; - case io.github.dfa1.vortex.fbs.PType.F64 -> PType.F64; + case io.github.dfa1.vortex.fbs.FbsPType.U8 -> PType.U8; + case io.github.dfa1.vortex.fbs.FbsPType.U16 -> PType.U16; + case io.github.dfa1.vortex.fbs.FbsPType.U32 -> PType.U32; + case io.github.dfa1.vortex.fbs.FbsPType.U64 -> PType.U64; + case io.github.dfa1.vortex.fbs.FbsPType.I8 -> PType.I8; + case io.github.dfa1.vortex.fbs.FbsPType.I16 -> PType.I16; + case io.github.dfa1.vortex.fbs.FbsPType.I32 -> PType.I32; + case io.github.dfa1.vortex.fbs.FbsPType.I64 -> PType.I64; + case io.github.dfa1.vortex.fbs.FbsPType.F16 -> PType.F16; + case io.github.dfa1.vortex.fbs.FbsPType.F32 -> PType.F32; + case io.github.dfa1.vortex.fbs.FbsPType.F64 -> PType.F64; default -> throw new VortexException("unrecognized PType=" + fbsPType); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index a3bd09270..0c768a5be 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -42,8 +42,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; @@ -114,8 +112,8 @@ private static void collectFlats(Layout layout, List out) { } else if (layout.isChunked()) { // metadata[0] == 1 means children[0] is the per-chunk stats layout; skip it int start = (layout.metadata() != null - && layout.metadata().hasRemaining() - && layout.metadata().get(0) == 1) ? 1 : 0; + && layout.metadata().byteSize() > 0 + && layout.metadata().get(ValueLayout.JAVA_BYTE, 0) == 1) ? 1 : 0; for (int i = start; i < layout.children().size(); i++) { collectFlats(layout.children().get(i), out); } @@ -581,7 +579,7 @@ private Array decodeFlat(Layout flat, DType dtype, SegmentAllocator arena) { } private Array decodeDictLayout(Layout dictLayout, DType dtype, SegmentAllocator arena) { - ByteBuffer rawMeta = dictLayout.metadata(); + MemorySegment rawMeta = dictLayout.metadata(); // DictLayoutMetadata proto (Rust format): field 1 = codes_ptype (PType varint). // Read the varint directly to avoid field-number mismatch with the array-level DictMetadata proto. PType codesPType = readDictLayoutCodesPType(rawMeta); @@ -676,17 +674,16 @@ private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array }; } - private static PType readDictLayoutCodesPType(ByteBuffer rawMeta) { + private static PType readDictLayoutCodesPType(MemorySegment rawMeta) { // DictLayoutMetadata (Rust): field 1 = codes_ptype, wire type 0 (varint). // Tag byte = (field_number << 3) | wire_type = (1 << 3) | 0 = 0x08. // Proto3 omits field 1 when it holds the default value (0 = U8), so empty metadata means U8. - if (rawMeta == null || !rawMeta.hasRemaining()) { + if (rawMeta == null || rawMeta.byteSize() == 0) { return PType.U8; } - ByteBuffer buf = rawMeta.duplicate(); - byte tag = buf.get(); - if (tag == 0x08 && buf.hasRemaining()) { - int ordinal = buf.get() & 0xFF; + byte tag = rawMeta.get(ValueLayout.JAVA_BYTE, 0); + if (tag == 0x08 && rawMeta.byteSize() > 1) { + int ordinal = rawMeta.get(ValueLayout.JAVA_BYTE, 1) & 0xFF; PType[] values = PType.values(); if (ordinal < values.length) { return values[ordinal]; @@ -820,10 +817,9 @@ private ArrayStats readFlatStats(Layout flat) { return ArrayStats.empty(); } long fbStart = segLen - 4L - fbLen; - ByteBuffer fbBuf = IoBounds.slice(seg, fbStart, fbLen).asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); - var fbArray = io.github.dfa1.vortex.fbs.Array.getRootAsArray(fbBuf); + var fbArray = io.github.dfa1.vortex.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); - io.github.dfa1.vortex.fbs.ArrayNode root = fbArray.root(); + io.github.dfa1.vortex.fbs.FbsArrayNode root = fbArray.root(); if (root == null) { return ArrayStats.empty(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java index 382cf5da1..334839729 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.IoBounds; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.core.VortexFormat; -import io.github.dfa1.vortex.fbs.Postscript; +import io.github.dfa1.vortex.fbs.FbsPostscript; import java.io.IOException; import java.lang.foreign.Arena; @@ -13,8 +13,6 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; /// Handle to a remote Vortex file read via HTTP Range requests. /// @@ -105,10 +103,9 @@ public static VortexHttpReader open(URI uri, ReadRegistry registry, HttpClient c .formatted(trailer.postscriptLen(), TAIL_SIZE)); } - ByteBuffer postscriptBuf = IoBounds.slice(tailSeg, psOffInTail, trailer.postscriptLen()) - .asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + MemorySegment postscriptSeg = IoBounds.slice(tailSeg, psOffInTail, trailer.postscriptLen()); - var ps = Postscript.getRootAsPostscript(postscriptBuf); + var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); var footerSpec = ps.footer(); if (footerSpec == null) { @@ -120,9 +117,9 @@ public static VortexHttpReader open(URI uri, ReadRegistry registry, HttpClient c } var dtypeSpec = ps.dtype(); - ByteBuffer footerBuf = fetchBlob(footerSpec.offset(), footerSpec.length(), tailStart, tail, uri, client); - ByteBuffer layoutBuf = fetchBlob(layoutSpec.offset(), layoutSpec.length(), tailStart, tail, uri, client); - ByteBuffer dtypeBuf = (dtypeSpec != null && dtypeSpec.length() > 0) + MemorySegment footerBuf = fetchBlob(footerSpec.offset(), footerSpec.length(), tailStart, tail, uri, client); + MemorySegment layoutBuf = fetchBlob(layoutSpec.offset(), layoutSpec.length(), tailStart, tail, uri, client); + MemorySegment dtypeBuf = (dtypeSpec != null && dtypeSpec.length() > 0) ? fetchBlob(dtypeSpec.offset(), dtypeSpec.length(), tailStart, tail, uri, client) : null; @@ -205,21 +202,20 @@ private static byte[] fetchRange(URI uri, long from, long to, HttpClient client) } } - /// Returns a ByteBuffer for a blob at absolute file `offset` of `length` bytes. + /// Returns a MemorySegment for a blob at absolute file `offset` of `length` bytes. /// If the blob falls within the already-fetched `tail`, extracts it directly; /// otherwise fires an additional Range request. - private static ByteBuffer fetchBlob( + private static MemorySegment fetchBlob( long offset, long length, long tailStart, byte[] tail, URI uri, HttpClient client ) throws IOException { if (offset >= tailStart) { int relOffset = (int) (offset - tailStart); - return ByteBuffer.wrap(tail, relOffset, (int) length) - .slice().order(ByteOrder.LITTLE_ENDIAN); + return MemorySegment.ofArray(tail).asSlice(relOffset, length); } byte[] bytes = fetchRange(uri, offset, offset + length - 1, client); - return ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + return MemorySegment.ofArray(bytes); } @Override diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index 8046d621f..76490dd51 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -9,7 +9,7 @@ import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; +import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -81,12 +81,11 @@ private static VortexReader parse( Trailer trailer = Trailer.parse(trailerSeg, bodyBytes); long postscriptOffset = bodyBytes - trailer.postscriptLen(); - var postscriptBuf = IoBounds.slice(seg, postscriptOffset, trailer.postscriptLen()) - .asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + var postscriptSeg = IoBounds.slice(seg, postscriptOffset, trailer.postscriptLen()); PostscriptParser.ParsedFile parsed; try { - parsed = PostscriptParser.parse(postscriptBuf, seg, size); + parsed = PostscriptParser.parse(postscriptSeg, seg, size); } catch (VortexException e) { throw e; } catch (RuntimeException e) { @@ -107,8 +106,8 @@ private static void collectFlats(Layout layout, List out) { collectFlats(layout.children().getFirst(), out); } else if (layout.isChunked()) { int start = (layout.metadata() != null - && layout.metadata().hasRemaining() - && layout.metadata().get(0) == 1) ? 1 : 0; + && layout.metadata().byteSize() > 0 + && layout.metadata().get(ValueLayout.JAVA_BYTE, 0) == 1) ? 1 : 0; for (int i = start; i < layout.children().size(); i++) { collectFlats(layout.children().get(i), out); } @@ -228,8 +227,7 @@ private ArrayStats readFlatStats(Layout flat) { return ArrayStats.empty(); } long fbStart = segLen - 4L - fbLen; - var fbBuf = IoBounds.slice(seg, fbStart, fbLen).asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); - var fbArray = io.github.dfa1.vortex.fbs.Array.getRootAsArray(fbBuf); + var fbArray = io.github.dfa1.vortex.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); var root = fbArray.root(); if (root == null) { return ArrayStats.empty(); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/UnknownArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/UnknownArray.java index c7a9dbc54..ddd1deb2d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/UnknownArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/UnknownArray.java @@ -5,7 +5,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.nio.ByteBuffer; /// Opaque passthrough array for encodings unknown to this reader. /// @@ -27,7 +26,7 @@ public record UnknownArray( String encodingId, DType dtype, long length, - ByteBuffer metadata, + MemorySegment metadata, MemorySegment[] buffers, Array[] children ) implements Array { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index 8bc2f4aa1..f940b9ee4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ALPMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyAlpDoubleArray; import io.github.dfa1.vortex.reader.array.LazyAlpFloatArray; @@ -18,7 +18,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.alp`. public final class AlpEncodingDecoder implements EncodingDecoder { @@ -38,14 +37,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - ALPMetadata meta; - if (rawMeta == null || !rawMeta.hasRemaining()) { - meta = new ALPMetadata(0, 0, null); + MemorySegment rawMeta = ctx.metadata(); + ProtoALPMetadata meta; + if (rawMeta == null || rawMeta.byteSize() == 0) { + meta = new ProtoALPMetadata(0, 0, null); } else { try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = ALPMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoALPMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_ALP, "invalid metadata", e); } @@ -67,7 +66,7 @@ public Array decode(DecodeContext ctx) { }; } - private static Array decodeF64(DecodeContext ctx, ALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { // Decode formula mirrors the Rust reference (`ALPFloat::decode_single`): two-step // `encoded * F10[f] * IF10[e]`. A pre-multiplied `scale = F10[f] * IF10[e]` // gives different IEEE rounding for non-trivial `expF`, breaking round-trip with @@ -101,7 +100,7 @@ private static Array decodeF64(DecodeContext ctx, ALPMetadata meta, int expE, in return new MaterializedDoubleArray(ctx.dtype(), n, buf.asReadOnly()); } - private static Array decodeF32(DecodeContext ctx, ALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { float df = F10_F32[expF]; float de = IF10_F32[expE]; MemorySegment src = ctx.decodeChildSegment(0, DType.I32, n); @@ -131,7 +130,7 @@ private static Array decodeF32(DecodeContext ctx, ALPMetadata meta, int expE, in return new MaterializedFloatArray(ctx.dtype(), n, buf.asReadOnly()); } - private static void applyPatches(DecodeContext ctx, PatchesMetadata pm, MemorySegment out, int elemBytes) { + private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, MemorySegment out, int elemBytes) { long numPatches = pm.len(); long offset = pm.offset(); PType idxPtype = PType.fromOrdinal(pm.indices_ptype().value()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java index 35d569eab..7dfc46511 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ALPRDMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPRDMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LazyAlpRdDoubleArray; @@ -17,7 +17,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.alprd`. public final class AlpRdEncodingDecoder implements EncodingDecoder { @@ -33,7 +32,7 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ALPRDMetadata meta = parseMeta(ctx); + ProtoALPRDMetadata meta = parseMeta(ctx); if (!(ctx.dtype() instanceof DType.Primitive p)) { throw new VortexException(EncodingId.VORTEX_ALPRD, @@ -85,7 +84,7 @@ private record Patches(Array indices, short[] leftValues, long offset) { static final Patches EMPTY = new Patches(null, new short[0], 0L); } - private static Patches decodePatches(DecodeContext ctx, PatchesMetadata pm) { + private static Patches decodePatches(DecodeContext ctx, ProtoPatchesMetadata pm) { if (pm == null || pm.len() == 0) { return Patches.EMPTY; } @@ -108,15 +107,15 @@ private static Patches decodePatches(DecodeContext ctx, PatchesMetadata pm) { return new Patches(idxData, leftValues, offset); } - private static ALPRDMetadata parseMeta(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - if (rawMeta == null || !rawMeta.hasRemaining()) { - return new ALPRDMetadata(0, 0, java.util.List.of(), - io.github.dfa1.vortex.proto.PType.fromValue(PType.U16.ordinal()), null); + private static ProtoALPRDMetadata parseMeta(DecodeContext ctx) { + MemorySegment rawMeta = ctx.metadata(); + if (rawMeta == null || rawMeta.byteSize() == 0) { + return new ProtoALPRDMetadata(0, 0, java.util.List.of(), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U16.ordinal()), null); } try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - return ALPRDMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + return ProtoALPRDMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_ALPRD, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java index 2a0237462..3726b3cd9 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; /// Encoded array node as stored in a Flat layout segment. /// In-file representation before decoding; mirrors the Go ArrayNode struct. @@ -21,13 +21,13 @@ public sealed interface ArrayNode permits KnownArrayNode, UnknownArrayNode { /// @param children child nodes /// @param bufferIndices segment buffer indices for this node /// @return a [KnownArrayNode] with the given fields - static ArrayNode of(EncodingId encodingId, ByteBuffer metadata, ArrayNode[] children, + static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children, int[] bufferIndices) { return new KnownArrayNode(encodingId, metadata, children, bufferIndices); } /// @return encoding-specific metadata bytes, or `null` - ByteBuffer metadata(); + MemorySegment metadata(); /// @return child nodes ArrayNode[] children(); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index ed26f0242..158078924 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.BitPackedMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedIntArray; @@ -16,7 +16,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `fastlanes.bitpacked`. public final class BitpackedEncodingDecoder implements EncodingDecoder { @@ -33,18 +32,18 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - // proto3 elides default-valued fields, so BitPackedMetadata(0, 0, null) serialises + MemorySegment rawMeta = ctx.metadata(); + // proto3 elides default-valued fields, so ProtoBitPackedMetadata(0, 0, null) serialises // to a 0-byte payload and the writer skips the empty vector. Treat absent metadata // as all-defaults rather than rejecting — happens when bit_width=0 (constant // residuals nested under FoR / RLE). - BitPackedMetadata meta; - if (rawMeta == null || !rawMeta.hasRemaining()) { - meta = new BitPackedMetadata(0, 0, null); + ProtoBitPackedMetadata meta; + if (rawMeta == null || rawMeta.byteSize() == 0) { + meta = new ProtoBitPackedMetadata(0, 0, null); } else { try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = BitPackedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoBitPackedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.FASTLANES_BITPACKED, "invalid metadata", e); } @@ -461,7 +460,7 @@ private static void unpackLoop64(MemorySegment buf, int bitWidth, int offset, lo } } - private static void applyPatches(DecodeContext ctx, PatchesMetadata pm, + private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, MemorySegment out, int elemBytes) { long numPatches = pm.len(); if (numPatches == 0) { @@ -497,7 +496,7 @@ private static long readUnsignedIdx(MemorySegment seg, long off, PType ptype) { }; } - private static PType ptypeFromProto(io.github.dfa1.vortex.proto.PType proto) { + private static PType ptypeFromProto(io.github.dfa1.vortex.proto.ProtoPType proto) { return PType.fromOrdinal(proto.value()); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java index b247d3cc0..d2cce624e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyConstantBoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; @@ -38,9 +38,9 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { MemorySegment scalarBuf = ctx.buffer(0); - ScalarValue scalar; + ProtoScalarValue scalar; try { - scalar = ScalarValue.decode(scalarBuf, 0, scalarBuf.byteSize()); + scalar = ProtoScalarValue.decode(scalarBuf, 0, scalarBuf.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "invalid scalar value", e); } @@ -50,7 +50,7 @@ public Array decode(DecodeContext ctx) { /// Builds the constant array for `scalar` interpreted as `dtype`, broadcast to `n` rows. /// Recurses for Extension (its storage dtype) and Variant (the wrapped inner scalar). - private static Array arrayFromScalar(DecodeContext ctx, ScalarValue scalar, DType dtype, long n) { + private static Array arrayFromScalar(DecodeContext ctx, ProtoScalarValue scalar, DType dtype, long n) { if (dtype instanceof DType.Null) { return new NullArray(dtype, n); } @@ -58,7 +58,7 @@ private static Array arrayFromScalar(DecodeContext ctx, ScalarValue scalar, DTyp // A constant variant wraps a typed inner scalar (Scalar::variant(inner)); the // physical storage is the inner-typed constant array. The VariantArray wrapper // re-applies the logical Variant dtype. - io.github.dfa1.vortex.proto.Scalar inner = scalar.variant_value(); + io.github.dfa1.vortex.proto.ProtoScalar inner = scalar.variant_value(); if (inner == null || inner.value() == null) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "constant variant missing variant_value"); } @@ -99,7 +99,7 @@ private static Array arrayFromScalar(DecodeContext ctx, ScalarValue scalar, DTyp /// @param scalar the constant scalar value /// @param n row count /// @return a lazy constant array of length `n` - private static Array constantPrimitive(DType outDtype, PType ptype, ScalarValue scalar, long n) { + private static Array constantPrimitive(DType outDtype, PType ptype, ProtoScalarValue scalar, long n) { long rawBits = scalarToRawBits(scalar, ptype); return switch (ptype) { case I64, U64 -> new LazyConstantLongArray(outDtype, n, rawBits); @@ -112,7 +112,7 @@ private static Array constantPrimitive(DType outDtype, PType ptype, ScalarValue }; } - private static Array decodeDecimal(DType dtype, ScalarValue scalar, long n) { + private static Array decodeDecimal(DType dtype, ProtoScalarValue scalar, long n) { byte[] elemBytes = scalar.bytes_value(); int elemLen = elemBytes.length; // Decode the single scalar value via LazyDecimalArray (reuses its LE byte-order logic), @@ -121,12 +121,12 @@ private static Array decodeDecimal(DType dtype, ScalarValue scalar, long n) { return new LazyConstantDecimalArray(dtype, n, value, elemLen); } - private static Array decodeBool(DType dtype, ScalarValue scalar, long n) { + private static Array decodeBool(DType dtype, ProtoScalarValue scalar, long n) { boolean value = scalar.bool_value() != null && scalar.bool_value(); return new LazyConstantBoolArray(dtype, n, value); } - private static Array decodeString(DecodeContext ctx, ScalarValue scalar, DType dtype, long n) { + private static Array decodeString(DecodeContext ctx, ProtoScalarValue scalar, DType dtype, long n) { byte[] strBytes = scalar.string_value() != null ? scalar.string_value().getBytes(StandardCharsets.UTF_8) : (scalar.bytes_value() != null ? scalar.bytes_value() : new byte[0]); @@ -146,7 +146,7 @@ private static Array decodeString(DecodeContext ctx, ScalarValue scalar, DType d return new VarBinArray.OffsetMode(dtype, n, bytesSeg.asReadOnly(), offsetsSeg.asReadOnly(), PType.I32); } - private static long scalarToRawBits(ScalarValue scalar, PType ptype) { + private static long scalarToRawBits(ProtoScalarValue scalar, PType ptype) { if (scalar.int64_value() != null) { return scalar.int64_value(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoder.java index df0644718..a7ae5b55b 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoder.java @@ -5,13 +5,12 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.TimeUnit; -import io.github.dfa1.vortex.proto.DateTimePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDateTimePartsMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyDateTimePartsLongArray; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.datetimeparts`. /// @@ -35,14 +34,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer meta = ctx.metadata(); - if (meta == null || meta.remaining() == 0) { + MemorySegment meta = ctx.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "missing metadata"); } - DateTimePartsMetadata decoded; + ProtoDateTimePartsMetadata decoded; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(meta.duplicate()); - decoded = DateTimePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = meta; + decoded = ProtoDateTimePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "invalid metadata: " + e.getMessage()); } @@ -71,12 +70,12 @@ public Array decode(DecodeContext ctx) { /// `1` when the unit is [TimeUnit#Days] (days carry no sub-second /// component; seconds and subseconds children are expected to be zero). private static long readUnitsPerSecond(DType.Extension ext) { - ByteBuffer extMeta = ext.metadata(); - if (extMeta == null || !extMeta.hasRemaining()) { + MemorySegment extMeta = ext.metadata(); + if (extMeta == null || extMeta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "extension " + ext.extensionId() + " missing TimeUnit metadata byte"); } - TimeUnit unit = TimeUnit.fromTag(extMeta.get(extMeta.position())); + TimeUnit unit = TimeUnit.fromTag(extMeta.get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)); return unit == TimeUnit.Days ? 1L : unit.divisor(); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalBytePartsEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalBytePartsEncodingDecoder.java index 2893c8e53..8dabbf405 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalBytePartsEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalBytePartsEncodingDecoder.java @@ -6,11 +6,10 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyDecimalBytePartsArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.DecimalBytePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDecimalBytePartsMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.decimal_byte_parts`. public final class DecimalBytePartsEncodingDecoder implements EncodingDecoder { @@ -26,14 +25,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer meta = ctx.metadata(); - if (meta == null || meta.remaining() == 0) { + MemorySegment meta = ctx.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DECIMAL_BYTE_PARTS, "missing metadata"); } - DecimalBytePartsMetadata decoded; + ProtoDecimalBytePartsMetadata decoded; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(meta.duplicate()); - decoded = DecimalBytePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = meta; + decoded = ProtoDecimalBytePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_DECIMAL_BYTE_PARTS, "invalid metadata: " + e.getMessage()); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoder.java index baa8dee6f..73721c4f3 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoder.java @@ -4,11 +4,10 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyDecimalArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.DecimalMetadata; +import io.github.dfa1.vortex.proto.ProtoDecimalMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.decimal`. public final class DecimalEncodingDecoder implements EncodingDecoder { @@ -24,14 +23,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer meta = ctx.metadata(); - if (meta == null || meta.remaining() == 0) { + MemorySegment meta = ctx.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DECIMAL, "missing metadata"); } - DecimalMetadata decoded; + ProtoDecimalMetadata decoded; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(meta.duplicate()); - decoded = DecimalMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = meta; + decoded = ProtoDecimalMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_DECIMAL, "invalid metadata: " + e.getMessage()); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java index 5b38ee839..4c02a1170 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java @@ -6,7 +6,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.nio.ByteBuffer; /// Decoding context passed to each [EncodingDecoder]. /// @@ -101,8 +100,8 @@ public MemorySegment buffer(int i) { /// Returns the encoding-specific metadata bytes for this node, or `null` if absent. /// - /// @return the metadata [ByteBuffer], or `null` - public ByteBuffer metadata() { + /// @return the metadata [MemorySegment], or `null` + public MemorySegment metadata() { return node.metadata(); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index 6a01bc238..5f4f1f555 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.encoding.FastLanes; import io.github.dfa1.vortex.encoding.PrimitiveArrays; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.DeltaMetadata; +import io.github.dfa1.vortex.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedIntArray; @@ -17,7 +17,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `fastlanes.delta`. public final class DeltaEncodingDecoder implements EncodingDecoder { @@ -33,14 +32,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - DeltaMetadata meta; - if (rawMeta == null || !rawMeta.hasRemaining()) { - meta = new DeltaMetadata(0L, 0); + MemorySegment rawMeta = ctx.metadata(); + ProtoDeltaMetadata meta; + if (rawMeta == null || rawMeta.byteSize() == 0) { + meta = new ProtoDeltaMetadata(0L, 0); } else { try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = DeltaMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoDeltaMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.FASTLANES_DELTA, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 1c4030feb..55deab3dc 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.DictMetadata; +import io.github.dfa1.vortex.proto.ProtoDictMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; @@ -18,7 +18,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.dict`. /// @@ -45,29 +44,29 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer meta = ctx.metadata(); + MemorySegment meta = ctx.metadata(); if (ctx.dtype() instanceof DType.Utf8) { if (ctx.node().children().length == 0) { - if (meta == null || !meta.hasRemaining()) { + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DICT, "missing metadata for legacy utf8 dict"); } return decodeUtf8DictLegacy(ctx, meta); } - if (meta == null || !meta.hasRemaining()) { + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DICT, "missing metadata for utf8 dict"); } - return decodeUtf8DictProto(ctx, meta.duplicate()); + return decodeUtf8DictProto(ctx, meta); } - if (meta == null || !meta.hasRemaining()) { + if (meta == null || meta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_DICT, "missing metadata"); } - if (meta.remaining() == 1) { - return decodeLegacyJava(ctx, meta.get(0)); + if (meta.byteSize() == 1) { + return decodeLegacyJava(ctx, meta.get(ValueLayout.JAVA_BYTE, 0)); } - return decodeRustProto(ctx, meta.duplicate()); + return decodeRustProto(ctx, meta); } private static Array decodeLegacyJava(DecodeContext ctx, byte codeTypeByte) { @@ -91,11 +90,11 @@ private static Array decodeLegacyJava(DecodeContext ctx, byte codeTypeByte) { return typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); } - private static Array decodeRustProto(DecodeContext ctx, ByteBuffer metaBuf) { - DictMetadata meta; + private static Array decodeRustProto(DecodeContext ctx, MemorySegment metaBuf) { + ProtoDictMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(metaBuf); - meta = DictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = metaBuf; + meta = ProtoDictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_DICT, "invalid proto metadata", e); } @@ -120,8 +119,8 @@ private static Array decodeRustProto(DecodeContext ctx, ByteBuffer metaBuf) { return typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); } - private static Array decodeUtf8DictLegacy(DecodeContext ctx, ByteBuffer meta) { - PType codePType = PType.fromOrdinal(Byte.toUnsignedInt(meta.get(0))); + private static Array decodeUtf8DictLegacy(DecodeContext ctx, MemorySegment meta) { + PType codePType = PType.fromOrdinal(Byte.toUnsignedInt(meta.get(ValueLayout.JAVA_BYTE, 0))); long n = ctx.rowCount(); MemorySegment dictBytes = ctx.buffer(0); @@ -133,11 +132,11 @@ private static Array decodeUtf8DictLegacy(DecodeContext ctx, ByteBuffer meta) { codes, codePType); } - private static Array decodeUtf8DictProto(DecodeContext ctx, ByteBuffer metaBuf) { - DictMetadata meta; + private static Array decodeUtf8DictProto(DecodeContext ctx, MemorySegment metaBuf) { + ProtoDictMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(metaBuf); - meta = DictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = metaBuf; + meta = ProtoDictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_DICT, "invalid utf8 dict proto metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FrameOfReferenceEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FrameOfReferenceEncodingDecoder.java index 0fbf186e0..78b75025c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FrameOfReferenceEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FrameOfReferenceEncodingDecoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyForByteArray; @@ -14,7 +14,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `fastlanes.for` (Frame of Reference). public final class FrameOfReferenceEncodingDecoder implements EncodingDecoder { @@ -30,14 +29,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - if (rawMeta == null || !rawMeta.hasRemaining()) { + MemorySegment rawMeta = ctx.metadata(); + if (rawMeta == null || rawMeta.byteSize() == 0) { throw new VortexException(EncodingId.FASTLANES_FOR, "missing metadata"); } - ScalarValue scalar; + ProtoScalarValue scalar; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - scalar = ScalarValue.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + scalar = ProtoScalarValue.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.FASTLANES_FOR, "invalid metadata", e); } @@ -72,7 +71,7 @@ public Array decode(DecodeContext ctx) { return validity != null ? new MaskedArray(result, validity) : result; } - private static long referenceValue(ScalarValue scalar) { + private static long referenceValue(ProtoScalarValue scalar) { if (scalar.int64_value() != null) { return scalar.int64_value(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java index d6064a7cf..68d75a5d4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java @@ -7,12 +7,11 @@ import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.FSSTMetadata; +import io.github.dfa1.vortex.proto.ProtoFSSTMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.fsst`. public final class FsstEncodingDecoder implements EncodingDecoder { @@ -30,14 +29,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); + MemorySegment rawMeta = ctx.metadata(); if (rawMeta == null) { throw new VortexException(EncodingId.VORTEX_FSST, "missing metadata"); } - FSSTMetadata meta; + ProtoFSSTMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = FSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoFSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_FSST, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java index 172ae253f..b8678910d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; /// Array node whose encoding id is well-known to this build (an [EncodingId] enum constant). /// @@ -13,7 +13,7 @@ @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record KnownArrayNode( EncodingId encodingId, - ByteBuffer metadata, + MemorySegment metadata, ArrayNode[] children, int[] bufferIndices ) implements ArrayNode { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListEncodingDecoder.java index bc8ab41cc..4d0b3c94e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListEncodingDecoder.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.ListArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ListMetadata; +import io.github.dfa1.vortex.proto.ProtoListMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -36,10 +36,10 @@ public Array decode(DecodeContext ctx) { "expected 2 or 3 children, got " + nchildren); } - ListMetadata meta; + ProtoListMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(ctx.metadata().duplicate()); - meta = ListMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = ctx.metadata(); + meta = ProtoListMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_LIST, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListViewEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListViewEncodingDecoder.java index 5e3141c12..6c0b7705c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListViewEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ListViewEncodingDecoder.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.ListViewArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ListViewMetadata; +import io.github.dfa1.vortex.proto.ProtoListViewMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -36,10 +36,10 @@ public Array decode(DecodeContext ctx) { "expected 3 or 4 children, got " + nchildren); } - ListViewMetadata meta; + ProtoListViewMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(ctx.metadata().duplicate()); - meta = ListViewMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = ctx.metadata(); + meta = ProtoListViewMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_LISTVIEW, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java index bcfd5203f..c85cce9da 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.PatchedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchedMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; @@ -16,7 +16,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.patched`. public final class PatchedEncodingDecoder implements EncodingDecoder { @@ -32,8 +31,8 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - if (rawMeta == null || !rawMeta.hasRemaining()) { + MemorySegment rawMeta = ctx.metadata(); + if (rawMeta == null || rawMeta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_PATCHED, "missing metadata"); } @@ -41,8 +40,8 @@ public Array decode(DecodeContext ctx) { long nLanes; long offset; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - PatchedMetadata meta = PatchedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + ProtoPatchedMetadata meta = ProtoPatchedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); nPatches = Integer.toUnsignedLong(meta.n_patches()); nLanes = Integer.toUnsignedLong(meta.n_lanes()); offset = Integer.toUnsignedLong(meta.offset()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java index 63244195a..adc8fc507 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.PcoChunkInfo; -import io.github.dfa1.vortex.proto.PcoMetadata; +import io.github.dfa1.vortex.proto.ProtoPcoChunkInfo; +import io.github.dfa1.vortex.proto.ProtoPcoMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.MaskedArray; @@ -20,7 +20,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.ByteOrder; /// Read-only decoder for `vortex.pco` — port of pcodec. @@ -45,7 +44,7 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - PcoMetadata meta = parseMeta(ctx); + ProtoPcoMetadata meta = parseMeta(ctx); validateHeader(meta); DType dtype = ctx.dtype(); @@ -90,7 +89,7 @@ public Array decode(DecodeContext ctx) { int[] batchOffsetBits2 = new int[PcoTansDecoder.BATCH_N]; for (int c = 0; c < nChunks; c++) { - PcoChunkInfo chunkInfo = meta.chunks().get(c); + ProtoPcoChunkInfo chunkInfo = meta.chunks().get(c); MemorySegment chunkMetaBuf = ctx.buffer(c); // chunk metas at indices 0..nChunks-1 PcoChunkMeta chunkMeta = readChunkMeta(chunkMetaBuf, dtypeSize); @@ -743,21 +742,21 @@ private static PcoBin[] readBins(LeBitReader r, int nBins, int ansSizeLog, int d return bins; } - private static PcoMetadata parseMeta(DecodeContext ctx) { - ByteBuffer raw = ctx.metadata(); + private static ProtoPcoMetadata parseMeta(DecodeContext ctx) { + MemorySegment raw = ctx.metadata(); if (raw == null) { - throw new VortexException(EncodingId.VORTEX_PCO, "missing PcoMetadata"); + throw new VortexException(EncodingId.VORTEX_PCO, "missing ProtoPcoMetadata"); } try { - MemorySegment metaSeg = MemorySegment.ofBuffer(raw.duplicate()); - return PcoMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = raw; + return ProtoPcoMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_PCO, - "invalid PcoMetadata: " + e.getMessage()); + "invalid ProtoPcoMetadata: " + e.getMessage()); } } - private static void validateHeader(PcoMetadata meta) { + private static void validateHeader(ProtoPcoMetadata meta) { byte[] header = meta.header(); if (header.length < 2) { throw new VortexException(EncodingId.VORTEX_PCO, diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index 896eb63d5..518a3263d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.RLEMetadata; +import io.github.dfa1.vortex.proto.ProtoRLEMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; @@ -22,7 +22,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `fastlanes.rle`. public final class RleEncodingDecoder implements EncodingDecoder { @@ -40,11 +39,11 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - RLEMetadata meta; + MemorySegment rawMeta = ctx.metadata(); + ProtoRLEMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = RLEMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoRLEMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.FASTLANES_RLE, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index 0d0e3aadb..433a2891a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.RunEndMetadata; +import io.github.dfa1.vortex.proto.ProtoRunEndMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -24,7 +24,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.runend`. public final class RunEndEncodingDecoder implements EncodingDecoder { @@ -40,15 +39,15 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); + MemorySegment rawMeta = ctx.metadata(); if (rawMeta == null) { throw new VortexException(EncodingId.VORTEX_RUNEND, "missing metadata"); } - RunEndMetadata meta; + ProtoRunEndMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = RunEndMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoRunEndMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_RUNEND, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java index 0ec996ab4..bdb4a455d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SequenceMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSequenceMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; @@ -20,7 +20,6 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.sequence` — `A[i] = base + i * multiplier`. public final class SequenceEncodingDecoder implements EncodingDecoder { @@ -36,14 +35,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer metaBuf = ctx.metadata(); - if (metaBuf == null || !metaBuf.hasRemaining()) { + MemorySegment metaBuf = ctx.metadata(); + if (metaBuf == null || metaBuf.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "missing metadata"); } - SequenceMetadata meta; + ProtoSequenceMetadata meta; try { - MemorySegment seg = MemorySegment.ofBuffer(metaBuf.duplicate()); - meta = SequenceMetadata.decode(seg, 0, seg.byteSize()); + MemorySegment seg = metaBuf; + meta = ProtoSequenceMetadata.decode(seg, 0, seg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "invalid metadata", e); } @@ -63,7 +62,7 @@ public Array decode(DecodeContext ctx) { } private static Array decodeInteger( - SequenceMetadata meta, PType pt, long n, DType dtype, SegmentAllocator arena + ProtoSequenceMetadata meta, PType pt, long n, DType dtype, SegmentAllocator arena ) { long base = signedValue(meta.base()); long mul = signedValue(meta.multiplier()); @@ -88,7 +87,7 @@ private static Array decodeInteger( }; } - private static Array decodeF32(SequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { + private static Array decodeF32(ProtoSequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { float base = meta.base().f32_value(); float mul = meta.multiplier().f32_value(); MemorySegment seg = arena.allocate(n * 4L); @@ -98,7 +97,7 @@ private static Array decodeF32(SequenceMetadata meta, long n, DType dtype, Segme return new MaterializedFloatArray(dtype, n, seg); } - private static Array decodeF64(SequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { + private static Array decodeF64(ProtoSequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { double base = meta.base().f64_value(); double mul = meta.multiplier().f64_value(); MemorySegment seg = arena.allocate(n * 8L); @@ -108,7 +107,7 @@ private static Array decodeF64(SequenceMetadata meta, long n, DType dtype, Segme return new MaterializedDoubleArray(dtype, n, seg); } - private static Array decodeF16(SequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { + private static Array decodeF16(ProtoSequenceMetadata meta, long n, DType dtype, SegmentAllocator arena) { short baseShort = (short) meta.base().f16_value().longValue(); short mulShort = (short) meta.multiplier().f16_value().longValue(); float base = Float.float16ToFloat(baseShort); @@ -120,7 +119,7 @@ private static Array decodeF16(SequenceMetadata meta, long n, DType dtype, Segme return new MaterializedFloat16Array(dtype, n, seg); } - private static long signedValue(ScalarValue sv) { + private static long signedValue(ProtoScalarValue sv) { if (sv == null) { return 0L; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java index 9d152cac6..fc173a1d1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java @@ -5,9 +5,9 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SparseMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSparseMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -29,7 +29,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.sparse`. public final class SparseEncodingDecoder implements EncodingDecoder { @@ -45,19 +44,19 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); - if (rawMeta == null || !rawMeta.hasRemaining()) { + MemorySegment rawMeta = ctx.metadata(); + if (rawMeta == null || rawMeta.byteSize() == 0) { throw new VortexException(EncodingId.VORTEX_SPARSE, "missing metadata"); } - SparseMetadata sparseMeta; + ProtoSparseMetadata sparseMeta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - sparseMeta = SparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + sparseMeta = ProtoSparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_SPARSE, "invalid metadata", e); } - PatchesMetadata patches = sparseMeta.patches(); + ProtoPatchesMetadata patches = sparseMeta.patches(); long numPatches = patches.len(); long offset = patches.offset(); PType indicesPtype = PType.fromOrdinal(patches.indices_ptype().value()); @@ -83,9 +82,9 @@ public Array decode(DecodeContext ctx) { PType valuePtype = ((DType.Primitive) ctx.dtype()).ptype(); MemorySegment fillBuf = ctx.buffer(0); - ScalarValue fillScalar; + ProtoScalarValue fillScalar; try { - fillScalar = ScalarValue.decode(fillBuf, 0, fillBuf.byteSize()); + fillScalar = ProtoScalarValue.decode(fillBuf, 0, fillBuf.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_SPARSE, "invalid fill value", e); } @@ -186,7 +185,7 @@ private static long readUnsignedIdx(MemorySegment seg, long off, PType ptype) { }; } - private static long scalarToLong(ScalarValue scalar) { + private static long scalarToLong(ProtoScalarValue scalar) { if (scalar.int64_value() != null) { return scalar.int64_value(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java index 3b4aba1f6..da46cb3db 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.decode; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; /// Array node whose encoding id is not a recognised [io.github.dfa1.vortex.encoding.EncodingId]. /// Produced when a file uses an encoding this build does not know about. Decoded as @@ -14,7 +14,7 @@ @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record UnknownArrayNode( String rawEncodingId, - ByteBuffer metadata, + MemorySegment metadata, ArrayNode[] children, int[] bufferIndices ) implements ArrayNode { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java index 154222ecd..73f4ee5a5 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java @@ -6,11 +6,10 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.varbin`. public final class VarBinEncodingDecoder implements EncodingDecoder { @@ -26,14 +25,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); + MemorySegment rawMeta = ctx.metadata(); if (rawMeta == null) { throw new VortexException(EncodingId.VORTEX_VARBIN, "missing metadata"); } - VarBinMetadata meta; + ProtoVarBinMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = VarBinMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoVarBinMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_VARBIN, "invalid metadata", e); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java index e64baaaed..65226ec5c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java @@ -6,11 +6,10 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VariantArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.VariantMetadata; +import io.github.dfa1.vortex.proto.ProtoVariantMetadata; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -46,13 +45,13 @@ public Array decode(DecodeContext ctx) { return new VariantArray(ctx.dtype(), ctx.rowCount(), coreStorage, shredded); } - private static DType parseShreddedDtype(ByteBuffer rawMeta) { - if (rawMeta == null || !rawMeta.hasRemaining()) { + private static DType parseShreddedDtype(MemorySegment rawMeta) { + if (rawMeta == null || rawMeta.byteSize() == 0) { return null; } try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - VariantMetadata meta = VariantMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + ProtoVariantMetadata meta = ProtoVariantMetadata.decode(metaSeg, 0, metaSeg.byteSize()); if (meta.shredded_dtype() == null) { return null; } @@ -62,7 +61,7 @@ private static DType parseShreddedDtype(ByteBuffer rawMeta) { } } - static DType dtypeFromProto(io.github.dfa1.vortex.proto.DType proto) { + static DType dtypeFromProto(io.github.dfa1.vortex.proto.ProtoDType proto) { if (proto.null_() != null) { return new DType.Null(true); } @@ -91,7 +90,7 @@ static DType dtypeFromProto(io.github.dfa1.vortex.proto.DType proto) { var names = new ArrayList(s.names().size()); var types = new ArrayList(s.dtypes().size()); names.addAll(s.names()); - for (io.github.dfa1.vortex.proto.DType child : s.dtypes()) { + for (io.github.dfa1.vortex.proto.ProtoDType child : s.dtypes()) { types.add(dtypeFromProto(child)); } return new DType.Struct(List.copyOf(names), List.copyOf(types), s.nullable()); @@ -111,7 +110,7 @@ static DType dtypeFromProto(io.github.dfa1.vortex.proto.DType proto) { return new DType.Extension( proto.extension().id(), dtypeFromProto(proto.extension().storage_dtype()), - ByteBuffer.wrap(proto.extension().metadata() != null ? proto.extension().metadata() : new byte[0]).asReadOnlyBuffer(), + MemorySegment.ofArray(proto.extension().metadata() != null ? proto.extension().metadata() : new byte[0]).asReadOnly(), false); } if (proto.variant() != null) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java index 51de6d697..61ea00002 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ZstdMetadata; +import io.github.dfa1.vortex.proto.ProtoZstdMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.MaskedArray; @@ -24,7 +24,6 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; /// Read-only decoder for `vortex.zstd`. public final class ZstdEncodingDecoder implements EncodingDecoder { @@ -40,14 +39,14 @@ public EncodingId encodingId() { @Override public Array decode(DecodeContext ctx) { - ByteBuffer rawMeta = ctx.metadata(); + MemorySegment rawMeta = ctx.metadata(); if (rawMeta == null) { throw new VortexException(EncodingId.VORTEX_ZSTD, "missing metadata"); } - ZstdMetadata meta; + ProtoZstdMetadata meta; try { - MemorySegment metaSeg = MemorySegment.ofBuffer(rawMeta.duplicate()); - meta = ZstdMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = rawMeta; + meta = ProtoZstdMetadata.decode(metaSeg, 0, metaSeg.byteSize()); } catch (IOException e) { throw new VortexException(EncodingId.VORTEX_ZSTD, "invalid metadata", e); } @@ -146,7 +145,7 @@ private static VarBinArray buildScatteredVarBin( private static MemorySegment decompressFrames( DecodeContext ctx, - ZstdMetadata meta, + ProtoZstdMetadata meta, int frameCount, long totalUncompressed ) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java index 2ce1f2adc..61c7b4395 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java @@ -10,7 +10,6 @@ import io.github.dfa1.vortex.reader.ExtensionDecoder; -import java.nio.ByteBuffer; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; @@ -35,8 +34,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { // Rust vortex.date metadata: 1 byte = TimeUnit tag (Days = 4), required by Rust reader. - ByteBuffer meta = ByteBuffer.allocate(1); - meta.put(0, (byte) TimeUnit.Days.ordinal()); + java.lang.foreign.MemorySegment meta = java.lang.foreign.MemorySegment.ofArray(new byte[]{(byte) TimeUnit.Days.ordinal()}); return new DType.Extension( ExtensionId.VORTEX_DATE.id(), new DType.Primitive(PType.I32, nullable), diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionStorage.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionStorage.java index 4bb48bae5..86e1c7f3e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionStorage.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionStorage.java @@ -10,7 +10,6 @@ import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.encoding.TimeUnit; -import java.nio.ByteBuffer; import java.time.Instant; import java.util.Objects; @@ -52,11 +51,11 @@ public static long epochInteger(Array storage, long i) { /// @return decoded time unit /// @throws VortexException if the metadata is missing public static TimeUnit readUnit(DType.Extension ext) { - ByteBuffer meta = ext.metadata(); - if (meta == null || !meta.hasRemaining()) { + java.lang.foreign.MemorySegment meta = ext.metadata(); + if (meta == null || meta.byteSize() == 0) { throw new VortexException("missing TimeUnit metadata byte for " + ext.extensionId()); } - return TimeUnit.fromTag(meta.get(meta.position())); + return TimeUnit.fromTag(meta.get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)); } /// Constructs an [Instant] from a signed epoch count in the given unit. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java index bfa5f8bd0..6828f70b1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java @@ -11,7 +11,6 @@ import io.github.dfa1.vortex.reader.ExtensionDecoder; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -40,7 +39,7 @@ public DType.Extension dtype(boolean nullable) { return new DType.Extension( ExtensionId.VORTEX_UUID.id(), new DType.FixedSizeList(u8, 16, nullable), - ByteBuffer.allocate(0), + java.lang.foreign.MemorySegment.ofArray(new byte[0]), nullable); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java index 57a6b1a52..096d5a3df 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java @@ -1,12 +1,11 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -52,7 +51,7 @@ void nullFbs_returnsEmpty() { @Test void fbsWithNoMinOrMax_returnsEmpty() { // Given — ArrayStats table with no min/max byte vectors set - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs(null, null); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs(null, null); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -64,9 +63,9 @@ void fbsWithNoMinOrMax_returnsEmpty() { @Test void int64Scalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofInt64Value(-7L).encode(), - ScalarValue.ofInt64Value(42L).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofInt64Value(-7L).encode(), + ProtoScalarValue.ofInt64Value(42L).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -79,9 +78,9 @@ void int64Scalar_decodes() { @Test void uint64Scalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofUint64Value(0L).encode(), - ScalarValue.ofUint64Value(Long.MAX_VALUE).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofUint64Value(0L).encode(), + ProtoScalarValue.ofUint64Value(Long.MAX_VALUE).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -94,9 +93,9 @@ void uint64Scalar_decodes() { @Test void f32Scalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofF32Value(1.5f).encode(), - ScalarValue.ofF32Value(3.25f).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofF32Value(1.5f).encode(), + ProtoScalarValue.ofF32Value(3.25f).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -109,9 +108,9 @@ void f32Scalar_decodes() { @Test void f64Scalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofF64Value(-0.5).encode(), - ScalarValue.ofF64Value(99.875).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofF64Value(-0.5).encode(), + ProtoScalarValue.ofF64Value(99.875).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -124,9 +123,9 @@ void f64Scalar_decodes() { @Test void boolScalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofBoolValue(false).encode(), - ScalarValue.ofBoolValue(true).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofBoolValue(false).encode(), + ProtoScalarValue.ofBoolValue(true).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -139,9 +138,9 @@ void boolScalar_decodes() { @Test void stringScalar_decodes() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofStringValue("alpha").encode(), - ScalarValue.ofStringValue("omega").encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofStringValue("alpha").encode(), + ProtoScalarValue.ofStringValue("omega").encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -155,9 +154,9 @@ void stringScalar_decodes() { void bytesScalar_decodesAsUtf8String() { // Given — bytes scalar surfaces as UTF-8 String for stat display purposes. // This is the contract zone-map pruning relies on (string compare across bytes/utf8). - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofBytesValue("aa".getBytes()).encode(), - ScalarValue.ofBytesValue("zz".getBytes()).encode()); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofBytesValue("aa".getBytes()).encode(), + ProtoScalarValue.ofBytesValue("zz".getBytes()).encode()); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -170,8 +169,8 @@ void bytesScalar_decodesAsUtf8String() { @Test void minOnly_setsMaxToNull() { // Given - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs( - ScalarValue.ofInt64Value(1L).encode(), null); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofInt64Value(1L).encode(), null); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -184,7 +183,7 @@ void minOnly_setsMaxToNull() { @Test void emptyByteVector_treatedAsAbsent() { // Given — zero-length min vector is structurally present but carries no scalar - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs(new byte[0], new byte[0]); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs(new byte[0], new byte[0]); // When ArrayStats sut = ArrayStats.fromFbs(fbs); @@ -196,7 +195,7 @@ void emptyByteVector_treatedAsAbsent() { @Test void malformedScalarBytes_throwsVortexException() { // Given — varint tag with no continuation byte: ProtoReader hits EOF inside readVarint - io.github.dfa1.vortex.fbs.ArrayStats fbs = buildFbs(new byte[]{(byte) 0x80}, null); + io.github.dfa1.vortex.fbs.FbsArrayStats fbs = buildFbs(new byte[]{(byte) 0x80}, null); // When / Then assertThatThrownBy(() -> ArrayStats.fromFbs(fbs)) @@ -205,20 +204,19 @@ void malformedScalarBytes_throwsVortexException() { } } - private static io.github.dfa1.vortex.fbs.ArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) { - FlatBufferBuilder b = new FlatBufferBuilder(64); - int minOff = minBytes == null ? 0 : io.github.dfa1.vortex.fbs.ArrayStats.createMinVector(b, minBytes); - int maxOff = maxBytes == null ? 0 : io.github.dfa1.vortex.fbs.ArrayStats.createMaxVector(b, maxBytes); - io.github.dfa1.vortex.fbs.ArrayStats.startArrayStats(b); + private static io.github.dfa1.vortex.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) { + FbsBuilder b = new FbsBuilder(64); + int minOff = minBytes == null ? 0 : io.github.dfa1.vortex.fbs.FbsArrayStats.createMinVector(b, minBytes); + int maxOff = maxBytes == null ? 0 : io.github.dfa1.vortex.fbs.FbsArrayStats.createMaxVector(b, maxBytes); + io.github.dfa1.vortex.fbs.FbsArrayStats.startFbsArrayStats(b); if (minBytes != null) { - io.github.dfa1.vortex.fbs.ArrayStats.addMin(b, minOff); + io.github.dfa1.vortex.fbs.FbsArrayStats.addMin(b, minOff); } if (maxBytes != null) { - io.github.dfa1.vortex.fbs.ArrayStats.addMax(b, maxOff); + io.github.dfa1.vortex.fbs.FbsArrayStats.addMax(b, maxOff); } - int root = io.github.dfa1.vortex.fbs.ArrayStats.endArrayStats(b); + int root = io.github.dfa1.vortex.fbs.FbsArrayStats.endFbsArrayStats(b); b.finish(root); - ByteBuffer buf = b.dataBuffer(); - return io.github.dfa1.vortex.fbs.ArrayStats.getRootAsArrayStats(buf); + return io.github.dfa1.vortex.fbs.FbsArrayStats.getRootAsFbsArrayStats(b.dataSegment()); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentBoundsSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentBoundsSecurityTest.java index 13ff7f99d..e939c36c8 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentBoundsSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentBoundsSecurityTest.java @@ -1,12 +1,12 @@ package io.github.dfa1.vortex.reader; import io.github.dfa1.vortex.encoding.PTypeIO; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.fbs.Array; -import io.github.dfa1.vortex.fbs.ArrayNode; -import io.github.dfa1.vortex.fbs.Buffer; +import io.github.dfa1.vortex.fbs.FbsArray; +import io.github.dfa1.vortex.fbs.FbsArrayNode; +import io.github.dfa1.vortex.fbs.FbsBuffer; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -17,7 +17,7 @@ /// Adversarial tests for the flat-segment decode path's offset/length arithmetic. /// -/// A flat segment is `buffer_data... | FlatBuffer(Array) | u32 LE = FlatBuffer byte length`. +/// A flat segment is `buffer_data... | FlatBuffer(FbsArray) | u32 LE = FlatBuffer byte length`. /// Both the trailing length field and each buffer descriptor's offset/length come straight /// from untrusted file bytes. After ADR 0003 Phase E, every malformed value must surface as /// a [VortexException], never a raw `IndexOutOfBoundsException` from `MemorySegment.asSlice`. @@ -70,7 +70,7 @@ void negativeFbLen_throwsVortexException() { @Test void bufferDescriptorLengthPastSegment_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { - // Given a well-formed Array FlatBuffer whose single buffer descriptor claims a + // Given a well-formed FbsArray FlatBuffer whose single buffer descriptor claims a // 1 000 000-byte payload that cannot fit the actual segment. byte[] fb = arrayFlatBufferWithOneBuffer(1_000_000L); MemorySegment seg = arena.allocate(fb.length + 4L); @@ -83,20 +83,20 @@ void bufferDescriptorLengthPastSegment_throwsVortexException() { } } - /// Builds a minimal valid `Array` FlatBuffer with one buffer descriptor of the given length. + /// Builds a minimal valid `FbsArray` FlatBuffer with one buffer descriptor of the given length. private static byte[] arrayFlatBufferWithOneBuffer(long bufferLength) { - FlatBufferBuilder b = new FlatBufferBuilder(); + FbsBuilder b = new FbsBuilder(); - int rootChildren = ArrayNode.createChildrenVector(b, new int[0]); - int rootBuffers = ArrayNode.createBuffersVector(b, new int[]{0}); - int root = ArrayNode.createArrayNode(b, 0, 0, rootChildren, rootBuffers, 0); + int rootChildren = FbsArrayNode.createChildrenVector(b, new int[0]); + int rootBuffers = FbsArrayNode.createBuffersVector(b, new int[]{0}); + int root = FbsArrayNode.createFbsArrayNode(b, 0, 0, rootChildren, rootBuffers, 0); - Array.startBuffersVector(b, 1); - Buffer.createBuffer(b, 0, 0, 0, bufferLength); + FbsArray.startBuffersVector(b, 1); + FbsBuffer.createFbsBuffer(b, 0, 0, 0, bufferLength); int buffers = b.endVector(); - int array = Array.createArray(b, root, buffers); - Array.finishArrayBuffer(b, array); + int array = FbsArray.createFbsArray(b, root, buffers); + FbsArray.finishFbsArrayBuffer(b, array); return b.sizedByteArray(); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java index f17118ac7..0b810a807 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java @@ -1,9 +1,9 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbs.FbsArrayNode; +import io.github.dfa1.vortex.fbs.FbsBuffer; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.core.DType; -import io.github.dfa1.vortex.fbs.ArrayNode; -import io.github.dfa1.vortex.fbs.Buffer; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.UnknownArray; import org.junit.jupiter.api.Test; @@ -53,18 +53,18 @@ void decode_unknownEncodingWithBufferPadding_returnsUnknownArray() { /// Builds an `Array` FlatBuffer with a single buffer descriptor of the given padding/length. private static byte[] arrayFlatBufferOneBuffer(int padding, long length) { - FlatBufferBuilder b = new FlatBufferBuilder(); + FbsBuilder b = new FbsBuilder(); - int rootChildren = ArrayNode.createChildrenVector(b, new int[0]); - int rootBuffers = ArrayNode.createBuffersVector(b, new int[]{0}); - int root = ArrayNode.createArrayNode(b, 0, 0, rootChildren, rootBuffers, 0); + int rootChildren = FbsArrayNode.createChildrenVector(b, new int[0]); + int rootBuffers = FbsArrayNode.createBuffersVector(b, new int[]{0}); + int root = FbsArrayNode.createFbsArrayNode(b, 0, 0, rootChildren, rootBuffers, 0); - io.github.dfa1.vortex.fbs.Array.startBuffersVector(b, 1); - Buffer.createBuffer(b, padding, 0, 0, length); + io.github.dfa1.vortex.fbs.FbsArray.startBuffersVector(b, 1); + FbsBuffer.createFbsBuffer(b, padding, 0, 0, length); int buffers = b.endVector(); - int array = io.github.dfa1.vortex.fbs.Array.createArray(b, root, buffers); - io.github.dfa1.vortex.fbs.Array.finishArrayBuffer(b, array); + int array = io.github.dfa1.vortex.fbs.FbsArray.createFbsArray(b, root, buffers); + io.github.dfa1.vortex.fbs.FbsArray.finishFbsArrayBuffer(b, array); return b.sizedByteArray(); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/LayoutDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/LayoutDepthBombSecurityTest.java index d9c401686..8c0b4992a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/LayoutDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/LayoutDepthBombSecurityTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import static io.github.dfa1.vortex.reader.MalformedFiles.buildFooter; import static io.github.dfa1.vortex.reader.MalformedFiles.buildI64Dtype; import static io.github.dfa1.vortex.reader.MalformedFiles.buildPostscript; @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.core.VortexFormat; -import io.github.dfa1.vortex.fbs.Layout; +import io.github.dfa1.vortex.fbs.FbsLayout; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -25,7 +25,7 @@ * [PostscriptParser]'s `convertLayout`. * *

The reader walks the layout tree recursively when materialising a file's - * `Layout` object. Without a depth cap a crafted file with thousands of + * `FbsLayout` object. Without a depth cap a crafted file with thousands of * nested children produces a [StackOverflowError] during `VortexReader.open`, * breaking the contract that every malformed input must surface as a [VortexException]. * @@ -51,7 +51,7 @@ void deeplyNestedLayout_throwsVortexException(@TempDir Path tmp) throws Exceptio // ── File builders ───────────────────────────────────────────────────────── /** - * Builds a .vtx file whose root Layout has `depth` levels of single-child nesting, + * Builds a .vtx file whose root FbsLayout has `depth` levels of single-child nesting, * each level reusing the same `vortex.flat` layout spec. */ private static Path buildDeeplyNestedFile(Path dir, int depth) throws Exception { @@ -68,15 +68,15 @@ private static Path buildDeeplyNestedFile(Path dir, int depth) throws Exception } private static ByteBuffer buildNestedLayout(int depth) { - var fbb = new FlatBufferBuilder(depth * 32); - int segV = Layout.createSegmentsVector(fbb, new long[]{0L}); + var fbb = new FbsBuilder(depth * 32); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0L}); // Build leaf first; FlatBuffer requires children be finished before parents. - int current = Layout.createLayout(fbb, 0, 1L, 0, 0, segV); + int current = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, segV); for (int i = 0; i < depth; i++) { - int childV = Layout.createChildrenVector(fbb, new int[]{current}); - current = Layout.createLayout(fbb, 0, 1L, 0, childV, 0); + int childV = FbsLayout.createChildrenVector(fbb, new int[]{current}); + current = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, childV, 0); } - Layout.finishLayoutBuffer(fbb, current); + FbsLayout.finishFbsLayoutBuffer(fbb, current); return slice(fbb); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/MalformedFiles.java b/reader/src/test/java/io/github/dfa1/vortex/reader/MalformedFiles.java index cb1b86bf5..c90d3c03c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/MalformedFiles.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/MalformedFiles.java @@ -1,15 +1,15 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; -import io.github.dfa1.vortex.fbs.ArraySpec; -import io.github.dfa1.vortex.fbs.Footer; -import io.github.dfa1.vortex.fbs.Layout; -import io.github.dfa1.vortex.fbs.LayoutSpec; -import io.github.dfa1.vortex.fbs.Postscript; -import io.github.dfa1.vortex.fbs.PostscriptSegment; -import io.github.dfa1.vortex.fbs.Primitive; -import io.github.dfa1.vortex.fbs.SegmentSpec; -import io.github.dfa1.vortex.fbs.Type; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbs.FbsArraySpec; +import io.github.dfa1.vortex.fbs.FbsFooter; +import io.github.dfa1.vortex.fbs.FbsLayout; +import io.github.dfa1.vortex.fbs.FbsLayoutSpec; +import io.github.dfa1.vortex.fbs.FbsPostscript; +import io.github.dfa1.vortex.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.fbs.FbsPrimitive; +import io.github.dfa1.vortex.fbs.FbsSegmentSpec; +import io.github.dfa1.vortex.fbs.FbsType; import java.nio.ByteBuffer; @@ -27,64 +27,64 @@ private MalformedFiles() { /// /// @return the finished DType FlatBuffer static ByteBuffer buildI64Dtype() { - var fbb = new FlatBufferBuilder(64); - int prim = Primitive.createPrimitive(fbb, io.github.dfa1.vortex.fbs.PType.I64, false); - int off = io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Primitive, prim); - io.github.dfa1.vortex.fbs.DType.finishDTypeBuffer(fbb, off); + var fbb = new FbsBuilder(64); + int prim = FbsPrimitive.createFbsPrimitive(fbb, io.github.dfa1.vortex.fbs.FbsPType.I64, false); + int off = io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, prim); + io.github.dfa1.vortex.fbs.FbsDType.finishFbsDTypeBuffer(fbb, off); return slice(fbb); } - /// Builds a Footer with the given array/layout spec ids and inline segment specs. + /// Builds a FbsFooter with the given array/layout spec ids and inline segment specs. /// /// @param arraySpecs encoding ids, one per array spec /// @param layoutSpecs layout ids, one per layout spec /// @param segOffsets per-segment byte offsets /// @param segLengths per-segment byte lengths - /// @return the finished Footer FlatBuffer + /// @return the finished FbsFooter FlatBuffer static ByteBuffer buildFooter( String[] arraySpecs, String[] layoutSpecs, long[] segOffsets, long[] segLengths) { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); int[] asOffs = new int[arraySpecs.length]; for (int i = 0; i < arraySpecs.length; i++) { - asOffs[i] = ArraySpec.createArraySpec(fbb, fbb.createString(arraySpecs[i])); + asOffs[i] = FbsArraySpec.createFbsArraySpec(fbb, fbb.createString(arraySpecs[i])); } - int asv = Footer.createArraySpecsVector(fbb, asOffs); + int asv = FbsFooter.createArraySpecsVector(fbb, asOffs); int[] lsOffs = new int[layoutSpecs.length]; for (int i = 0; i < layoutSpecs.length; i++) { - lsOffs[i] = LayoutSpec.createLayoutSpec(fbb, fbb.createString(layoutSpecs[i])); + lsOffs[i] = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(layoutSpecs[i])); } - int lsv = Footer.createLayoutSpecsVector(fbb, lsOffs); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, lsOffs); - // SegmentSpec is an inline struct — write in reverse order. - Footer.startSegmentSpecsVector(fbb, segOffsets.length); + // FbsSegmentSpec is an inline struct — write in reverse order. + FbsFooter.startSegmentSpecsVector(fbb, segOffsets.length); for (int i = segOffsets.length - 1; i >= 0; i--) { - SegmentSpec.createSegmentSpec(fbb, segOffsets[i], segLengths[i], 6, 0, 0); + FbsSegmentSpec.createFbsSegmentSpec(fbb, segOffsets[i], segLengths[i], 6, 0, 0); } int ssv = fbb.endVector(); - int footOff = Footer.createFooter(fbb, asv, lsv, ssv, 0, 0); + int footOff = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); fbb.finish(footOff); return slice(fbb); } - /// Builds a flat Layout referencing a single segment. + /// Builds a flat FbsLayout referencing a single segment. /// /// @param layoutSpecIdx index into the footer's layout-spec vector /// @param rowCount declared row count /// @param segIdx index into the footer's segment-spec vector - /// @return the finished Layout FlatBuffer + /// @return the finished FbsLayout FlatBuffer static ByteBuffer buildFlatLayout(int layoutSpecIdx, long rowCount, int segIdx) { - var fbb = new FlatBufferBuilder(128); - int segV = Layout.createSegmentsVector(fbb, new long[]{segIdx}); - int layoutOff = Layout.createLayout(fbb, layoutSpecIdx, rowCount, 0, 0, segV); - Layout.finishLayoutBuffer(fbb, layoutOff); + var fbb = new FbsBuilder(128); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{segIdx}); + int layoutOff = FbsLayout.createFbsLayout(fbb, layoutSpecIdx, rowCount, 0, 0, segV); + FbsLayout.finishFbsLayoutBuffer(fbb, layoutOff); return slice(fbb); } - /// Builds a Postscript pointing at the footer, dtype, and layout blobs. + /// Builds a FbsPostscript pointing at the footer, dtype, and layout blobs. /// /// @param footerOff footer byte offset /// @param footerLen footer byte length @@ -92,17 +92,17 @@ static ByteBuffer buildFlatLayout(int layoutSpecIdx, long rowCount, int segIdx) /// @param dtypeLen dtype byte length /// @param layoutOff layout byte offset /// @param layoutLen layout byte length - /// @return the finished Postscript FlatBuffer + /// @return the finished FbsPostscript FlatBuffer static ByteBuffer buildPostscript( long footerOff, int footerLen, long dtypeOff, int dtypeLen, long layoutOff, int layoutLen) { - var fbb = new FlatBufferBuilder(128); - int footSeg = PostscriptSegment.createPostscriptSegment(fbb, footerOff, footerLen, 0, 0, 0); - int dtypeSeg = PostscriptSegment.createPostscriptSegment(fbb, dtypeOff, dtypeLen, 0, 0, 0); - int layoutSeg = PostscriptSegment.createPostscriptSegment(fbb, layoutOff, layoutLen, 0, 0, 0); - int psOff = Postscript.createPostscript(fbb, dtypeSeg, layoutSeg, 0, footSeg); - Postscript.finishPostscriptBuffer(fbb, psOff); + var fbb = new FbsBuilder(128); + int footSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, footerOff, footerLen, 0, 0, 0); + int dtypeSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, dtypeOff, dtypeLen, 0, 0, 0); + int layoutSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, layoutOff, layoutLen, 0, 0, 0); + int psOff = FbsPostscript.createFbsPostscript(fbb, dtypeSeg, layoutSeg, 0, footSeg); + FbsPostscript.finishFbsPostscriptBuffer(fbb, psOff); return slice(fbb); } @@ -110,8 +110,7 @@ static ByteBuffer buildPostscript( /// /// @param fbb a builder whose root has been finished /// @return a [ByteBuffer] view of the finished bytes - static ByteBuffer slice(FlatBufferBuilder fbb) { - ByteBuffer data = fbb.dataBuffer(); - return data.slice(data.position(), data.remaining()); + static ByteBuffer slice(FbsBuilder fbb) { + return fbb.dataSegment().asByteBuffer().order(java.nio.ByteOrder.LITTLE_ENDIAN); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBigSegmentTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBigSegmentTest.java index e710d7cfa..558ec7e75 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBigSegmentTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBigSegmentTest.java @@ -1,8 +1,8 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; -import io.github.dfa1.vortex.fbs.ArraySpec; -import io.github.dfa1.vortex.fbs.LayoutSpec; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; +import io.github.dfa1.vortex.fbs.FbsArraySpec; +import io.github.dfa1.vortex.fbs.FbsLayoutSpec; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; @@ -12,24 +12,24 @@ class PostscriptParserBigSegmentTest { private static ByteBuffer buildMinimalFooter(long segOffset, long segLength) { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); // Empty array_specs + layout_specs vectors (required tables, no entries). - int asv = io.github.dfa1.vortex.fbs.Footer.createArraySpecsVector(fbb, new int[]{ - ArraySpec.createArraySpec(fbb, fbb.createString("vortex.flat")) + int asv = io.github.dfa1.vortex.fbs.FbsFooter.createArraySpecsVector(fbb, new int[]{ + FbsArraySpec.createFbsArraySpec(fbb, fbb.createString("vortex.flat")) }); - int lsv = io.github.dfa1.vortex.fbs.Footer.createLayoutSpecsVector(fbb, new int[]{ - LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.flat")) + int lsv = io.github.dfa1.vortex.fbs.FbsFooter.createLayoutSpecsVector(fbb, new int[]{ + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.flat")) }); // One segment_spec with the big length. - io.github.dfa1.vortex.fbs.Footer.startSegmentSpecsVector(fbb, 1); - io.github.dfa1.vortex.fbs.SegmentSpec.createSegmentSpec(fbb, segOffset, segLength, 6, 0, 0); + io.github.dfa1.vortex.fbs.FbsFooter.startSegmentSpecsVector(fbb, 1); + io.github.dfa1.vortex.fbs.FbsSegmentSpec.createFbsSegmentSpec(fbb, segOffset, segLength, 6, 0, 0); int ssv = fbb.endVector(); - int off = io.github.dfa1.vortex.fbs.Footer.createFooter(fbb, asv, lsv, ssv, 0, 0); + int off = io.github.dfa1.vortex.fbs.FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); fbb.finish(off); - return fbb.dataBuffer(); + return fbb.dataSegment().asByteBuffer().order(java.nio.ByteOrder.LITTLE_ENDIAN); } @Test @@ -40,8 +40,8 @@ void convertFooter_preservesSegmentLengthAbove2GB() { long bigLength = 0xC000_0000L; long bigOffset = 0x1_0000_0000L; // 4 GB into the file ByteBuffer fbsFooterBytes = buildMinimalFooter(bigOffset, bigLength); - io.github.dfa1.vortex.fbs.Footer fbsFooter = - io.github.dfa1.vortex.fbs.Footer.getRootAsFooter(fbsFooterBytes); + io.github.dfa1.vortex.fbs.FbsFooter fbsFooter = + io.github.dfa1.vortex.fbs.FbsFooter.getRootAsFbsFooter(java.lang.foreign.MemorySegment.ofBuffer(fbsFooterBytes)); // When Footer footer = PostscriptParser.convertFooter(fbsFooter); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java index 9bed8b660..5c98158e1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java @@ -1,19 +1,18 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.fbs.ArraySpec; -import io.github.dfa1.vortex.fbs.Footer; -import io.github.dfa1.vortex.fbs.Layout; -import io.github.dfa1.vortex.fbs.LayoutSpec; -import io.github.dfa1.vortex.fbs.Postscript; -import io.github.dfa1.vortex.fbs.PostscriptSegment; -import io.github.dfa1.vortex.fbs.Primitive; -import io.github.dfa1.vortex.fbs.Type; +import io.github.dfa1.vortex.fbs.FbsArraySpec; +import io.github.dfa1.vortex.fbs.FbsFooter; +import io.github.dfa1.vortex.fbs.FbsLayout; +import io.github.dfa1.vortex.fbs.FbsLayoutSpec; +import io.github.dfa1.vortex.fbs.FbsPostscript; +import io.github.dfa1.vortex.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.fbs.FbsPrimitive; +import io.github.dfa1.vortex.fbs.FbsType; import org.junit.jupiter.api.Test; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -44,12 +43,12 @@ private record Fixture(MemorySegment segment, long fileSize, } private static Fixture validFile() { - ByteBuffer layout = buildFlatLayout(); - ByteBuffer dtype = buildI64Dtype(); - ByteBuffer footer = buildFooter(); - int layoutLen = layout.remaining(); - int dtypeLen = dtype.remaining(); - int footerLen = footer.remaining(); + MemorySegment layout = buildFlatLayout(); + MemorySegment dtype = buildI64Dtype(); + MemorySegment footer = buildFooter(); + int layoutLen = (int) layout.byteSize(); + int dtypeLen = (int) dtype.byteSize(); + int footerLen = (int) footer.byteSize(); int layoutOff = 0; int dtypeOff = layoutOff + layoutLen; @@ -69,7 +68,7 @@ private static Fixture validFile() { void parse_validInBoundsBlobs_succeeds() { // Given — every blob pointer fits; footer ends exactly at EOF (length == fileSize - offset) Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.footerOff, f.footerLen, f.dtypeOff, f.dtypeLen, + MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.dtypeOff, f.dtypeLen, f.layoutOff, f.layoutLen); // When / Then — the largest legal footer range must not be rejected @@ -81,7 +80,7 @@ void parse_validInBoundsBlobs_succeeds() { void parse_footerBlobPastEof_throwsNamingFooter() { // Given — footer pointer one byte past EOF; everything else valid Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.fileSize + 1, f.footerLen, f.dtypeOff, f.dtypeLen, + MemorySegment ps = buildPostscript(f.fileSize + 1, f.footerLen, f.dtypeOff, f.dtypeLen, f.layoutOff, f.layoutLen); // When / Then — rejected by the footer-specific check, not the later slice @@ -94,7 +93,7 @@ void parse_footerBlobPastEof_throwsNamingFooter() { void parse_layoutBlobLengthOverrunsEof_throwsNamingLayout() { // Given — layout length reaches one byte past EOF (offset valid, offset + length > fileSize) Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.footerOff, f.footerLen, f.dtypeOff, f.dtypeLen, + MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.dtypeOff, f.dtypeLen, f.layoutOff, (int) (f.fileSize - f.layoutOff + 1)); // When / Then @@ -110,7 +109,7 @@ void parse_footerBlobLengthOverrunsEof_throwsNamingFooter() { // different answers — this is what kills the `fileSize - offset` → `+` math mutant that a // zero-offset overrun (the layout case) cannot distinguish. Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.footerOff, (int) (f.fileSize - f.footerOff + 1), + MemorySegment ps = buildPostscript(f.footerOff, (int) (f.fileSize - f.footerOff + 1), f.dtypeOff, f.dtypeLen, f.layoutOff, f.layoutLen); // When / Then @@ -123,7 +122,7 @@ void parse_footerBlobLengthOverrunsEof_throwsNamingFooter() { void parse_dtypeBlobPastEof_throwsNamingDtype() { // Given — dtype pointer past EOF with non-zero length, so the dtype check runs Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 1, f.dtypeLen, + MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 1, f.dtypeLen, f.layoutOff, f.layoutLen); // When / Then @@ -139,7 +138,7 @@ void parse_dtypeBlobLengthZero_skipsDtypeCheckEvenWithBadOffset() { // offset and the file still parses (dtype absent). Kills the `length > 0` boundary/negate // mutants that would otherwise run the check on an empty dtype. Fixture f = validFile(); - ByteBuffer ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 999, 0, + MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 999, 0, f.layoutOff, f.layoutLen); // When @@ -151,55 +150,53 @@ void parse_dtypeBlobLengthZero_skipsDtypeCheckEvenWithBadOffset() { // ── FlatBuffer blob builders (minimal, just enough to parse) ──────────────── - private static ByteBuffer buildFooter() { - var fbb = new FlatBufferBuilder(256); - int asv = Footer.createArraySpecsVector(fbb, new int[]{ - ArraySpec.createArraySpec(fbb, fbb.createString("vortex.primitive"))}); - int lsv = Footer.createLayoutSpecsVector(fbb, new int[]{ - LayoutSpec.createLayoutSpec(fbb, fbb.createString(io.github.dfa1.vortex.reader.Layout.FLAT))}); + private static MemorySegment buildFooter() { + var fbb = new FbsBuilder(256); + int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ + FbsArraySpec.createFbsArraySpec(fbb, fbb.createString("vortex.primitive"))}); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, new int[]{ + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(io.github.dfa1.vortex.reader.Layout.FLAT))}); // No segment_specs: validateSegmentSpecs has its own dedicated test; keep this fixture // focused on the blob-pointer bounds. - Footer.startSegmentSpecsVector(fbb, 0); + FbsFooter.startSegmentSpecsVector(fbb, 0); int ssv = fbb.endVector(); - int footOff = Footer.createFooter(fbb, asv, lsv, ssv, 0, 0); + int footOff = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); fbb.finish(footOff); return slice(fbb); } - private static ByteBuffer buildI64Dtype() { - var fbb = new FlatBufferBuilder(64); - int prim = Primitive.createPrimitive(fbb, io.github.dfa1.vortex.fbs.PType.I64, false); - int off = io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Primitive, prim); - io.github.dfa1.vortex.fbs.DType.finishDTypeBuffer(fbb, off); + private static MemorySegment buildI64Dtype() { + var fbb = new FbsBuilder(64); + int prim = FbsPrimitive.createFbsPrimitive(fbb, io.github.dfa1.vortex.fbs.FbsPType.I64, false); + int off = io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, prim); + io.github.dfa1.vortex.fbs.FbsDType.finishFbsDTypeBuffer(fbb, off); return slice(fbb); } - private static ByteBuffer buildFlatLayout() { - var fbb = new FlatBufferBuilder(128); - int segV = Layout.createSegmentsVector(fbb, new long[]{0}); - int layoutOff = Layout.createLayout(fbb, 0, 1L, 0, 0, segV); - Layout.finishLayoutBuffer(fbb, layoutOff); + private static MemorySegment buildFlatLayout() { + var fbb = new FbsBuilder(128); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + int layoutOff = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, segV); + FbsLayout.finishFbsLayoutBuffer(fbb, layoutOff); return slice(fbb); } - private static ByteBuffer buildPostscript( + private static MemorySegment buildPostscript( long footerOff, int footerLen, long dtypeOff, int dtypeLen, long layoutOff, int layoutLen) { - var fbb = new FlatBufferBuilder(128); - int footSeg = PostscriptSegment.createPostscriptSegment(fbb, footerOff, footerLen, 0, 0, 0); - int dtypeSeg = PostscriptSegment.createPostscriptSegment(fbb, dtypeOff, dtypeLen, 0, 0, 0); - int layoutSeg = PostscriptSegment.createPostscriptSegment(fbb, layoutOff, layoutLen, 0, 0, 0); - int psOff = Postscript.createPostscript(fbb, dtypeSeg, layoutSeg, 0, footSeg); - Postscript.finishPostscriptBuffer(fbb, psOff); + var fbb = new FbsBuilder(128); + int footSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, footerOff, footerLen, 0, 0, 0); + int dtypeSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, dtypeOff, dtypeLen, 0, 0, 0); + int layoutSeg = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, layoutOff, layoutLen, 0, 0, 0); + int psOff = FbsPostscript.createFbsPostscript(fbb, dtypeSeg, layoutSeg, 0, footSeg); + FbsPostscript.finishFbsPostscriptBuffer(fbb, psOff); return slice(fbb); } - private static ByteBuffer slice(FlatBufferBuilder fbb) { - ByteBuffer data = fbb.dataBuffer(); - return data.slice(data.position(), data.remaining()); + private static MemorySegment slice(FbsBuilder fbb) { + return fbb.dataSegment(); } - private static void copyInto(byte[] dst, int offset, ByteBuffer src) { - ByteBuffer dup = src.duplicate(); - dup.get(dst, offset, dup.remaining()); + private static void copyInto(byte[] dst, int offset, MemorySegment src) { + MemorySegment.copy(src, java.lang.foreign.ValueLayout.JAVA_BYTE, 0, dst, offset, (int) src.byteSize()); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java index 3a1cbbe5b..49239a0c0 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java @@ -1,19 +1,20 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import java.lang.foreign.MemorySegment; + +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; -import io.github.dfa1.vortex.fbs.ArraySpec; -import io.github.dfa1.vortex.fbs.Decimal; -import io.github.dfa1.vortex.fbs.Footer; -import io.github.dfa1.vortex.fbs.Layout; -import io.github.dfa1.vortex.fbs.LayoutSpec; -import io.github.dfa1.vortex.fbs.Type; +import io.github.dfa1.vortex.fbs.FbsArraySpec; +import io.github.dfa1.vortex.fbs.FbsDecimal; +import io.github.dfa1.vortex.fbs.FbsFooter; +import io.github.dfa1.vortex.fbs.FbsLayout; +import io.github.dfa1.vortex.fbs.FbsLayoutSpec; +import io.github.dfa1.vortex.fbs.FbsType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -26,13 +27,13 @@ /// legal value must parse, the first illegal value must throw a [VortexException]. class PostscriptParserParseBlobsBoundsTest { - // ── Layout encoding-index bound: encIdx < 0 || encIdx >= layoutSpecs.size() ── + // ── FbsLayout encoding-index bound: encIdx < 0 || encIdx >= layoutSpecs.size() ── @Test void parseBlobs_layoutEncodingIndex_atLastSpec_parses() { // Given — one layout spec, layout references index 0 (== size - 1, the last valid index) - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = flatLayout(0); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = flatLayout(0); // When / Then — the top valid index must be accepted (kills `>=` relaxed to `>`) assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -42,8 +43,8 @@ void parseBlobs_layoutEncodingIndex_atLastSpec_parses() { @Test void parseBlobs_layoutEncodingIndex_equalToSize_throws() { // Given — one layout spec (size 1), layout references index 1 (first out-of-range index) - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = flatLayout(1); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = flatLayout(1); // When / Then assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -51,15 +52,15 @@ void parseBlobs_layoutEncodingIndex_equalToSize_throws() { .hasMessageContaining("encoding index"); } - // ── Layout depth bound: depth > MAX_LAYOUT_DEPTH (64) ──────────────────────── + // ── FbsLayout depth bound: depth > MAX_LAYOUT_DEPTH (64) ──────────────────────── @Test void parseBlobs_layoutDepth_atLimit_parses() { // Given — a single-child chain whose deepest node sits at exactly MAX_LAYOUT_DEPTH. // convertLayout is called with depth == 64 there, and `64 > 64` is false. Kills the // `depth >` relaxed to `depth >=` mutant, which would reject the legal max-depth tree. - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH); // When / Then assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -69,8 +70,8 @@ void parseBlobs_layoutDepth_atLimit_parses() { @Test void parseBlobs_layoutDepth_oneOverLimit_throws() { // Given — one level deeper: the deepest node reaches depth 65, tripping `65 > 64` - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH + 1); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH + 1); // When / Then assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -78,7 +79,7 @@ void parseBlobs_layoutDepth_oneOverLimit_throws() { .hasMessageContaining("depth"); } - // ── Decimal precision bound: precision < 1 || precision > 38 ───────────────── + // ── FbsDecimal precision bound: precision < 1 || precision > 38 ───────────────── @ParameterizedTest @ValueSource(ints = {1, 38}) @@ -86,7 +87,7 @@ void parseBlobs_decimalPrecision_atEdges_parses(int precision) { // Given — precision at the inclusive edges 1 and 38 (scale 0 keeps the scale guard happy). // Kills `precision < 1` -> `<= 1` (would reject precision 1) and `precision > 38` -> // `>= 38` (would reject precision 38). - ByteBuffer dtype = decimalDtype(precision, (byte) 0); + MemorySegment dtype = decimalDtype(precision, (byte) 0); // When DType result = parseDtype(dtype); @@ -99,7 +100,7 @@ void parseBlobs_decimalPrecision_atEdges_parses(int precision) { @ValueSource(ints = {0, 39}) void parseBlobs_decimalPrecision_outOfRange_throws(int precision) { // Given — precision just outside [1, 38] - ByteBuffer dtype = decimalDtype(precision, (byte) 0); + MemorySegment dtype = decimalDtype(precision, (byte) 0); // When / Then assertThatThrownBy(() -> parseDtype(dtype)) @@ -107,7 +108,7 @@ void parseBlobs_decimalPrecision_outOfRange_throws(int precision) { .hasMessageContaining("precision"); } - // ── Decimal scale bound: scale < 0 || scale > precision ────────────────────── + // ── FbsDecimal scale bound: scale < 0 || scale > precision ────────────────────── @Test void parseBlobs_decimalScale_atEdges_parses() { @@ -123,7 +124,7 @@ void parseBlobs_decimalScale_atEdges_parses() { @Test void parseBlobs_decimalScale_abovePrecision_throws() { // Given — scale one past precision - ByteBuffer dtype = decimalDtype(10, (byte) 11); + MemorySegment dtype = decimalDtype(10, (byte) 11); // When / Then assertThatThrownBy(() -> parseDtype(dtype)) @@ -131,14 +132,14 @@ void parseBlobs_decimalScale_abovePrecision_throws() { .hasMessageContaining("scale"); } - // ── Layout metadata-size bound: metadata.remaining() > MAX_LAYOUT_METADATA_BYTES ── + // ── FbsLayout metadata-size bound: metadata.remaining() > MAX_LAYOUT_METADATA_BYTES ── @Test void parseBlobs_layoutMetadata_atLimit_parses() { // Given — metadata of exactly MAX_LAYOUT_METADATA_BYTES (the largest allowed). `> MAX` is // false at the limit, so it must parse; kills `>` relaxed to `>=`, which would reject it. - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = flatLayoutWithMetadata(PostscriptParser.MAX_LAYOUT_METADATA_BYTES); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = flatLayoutWithMetadata(PostscriptParser.MAX_LAYOUT_METADATA_BYTES); // When / Then assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -148,8 +149,8 @@ void parseBlobs_layoutMetadata_atLimit_parses() { @Test void parseBlobs_layoutMetadata_oneOverLimit_throws() { // Given — one byte past the cap - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = flatLayoutWithMetadata(PostscriptParser.MAX_LAYOUT_METADATA_BYTES + 1); + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = flatLayoutWithMetadata(PostscriptParser.MAX_LAYOUT_METADATA_BYTES + 1); // When / Then assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -161,68 +162,67 @@ void parseBlobs_layoutMetadata_oneOverLimit_throws() { /// Parses a dtype blob through the full parseBlobs path, paired with a minimal valid /// footer + flat layout so convertDType is reached. Returns the decoded [DType]. - private static DType parseDtype(ByteBuffer dtype) { - ByteBuffer footer = footerWithLayoutSpecs("vortex.flat"); - ByteBuffer layout = flatLayout(0); + private static DType parseDtype(MemorySegment dtype) { + MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); + MemorySegment layout = flatLayout(0); return PostscriptParser.parseBlobs(footer, layout, dtype).dtype(); } - private static ByteBuffer footerWithLayoutSpecs(String... layoutSpecs) { - var fbb = new FlatBufferBuilder(256); - int asv = Footer.createArraySpecsVector(fbb, new int[]{ - ArraySpec.createArraySpec(fbb, fbb.createString("vortex.primitive"))}); + private static MemorySegment footerWithLayoutSpecs(String... layoutSpecs) { + var fbb = new FbsBuilder(256); + int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ + FbsArraySpec.createFbsArraySpec(fbb, fbb.createString("vortex.primitive"))}); int[] ls = new int[layoutSpecs.length]; for (int i = 0; i < layoutSpecs.length; i++) { - ls[i] = LayoutSpec.createLayoutSpec(fbb, fbb.createString(layoutSpecs[i])); + ls[i] = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(layoutSpecs[i])); } - int lsv = Footer.createLayoutSpecsVector(fbb, ls); - Footer.startSegmentSpecsVector(fbb, 0); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, ls); + FbsFooter.startSegmentSpecsVector(fbb, 0); int ssv = fbb.endVector(); - int footOff = Footer.createFooter(fbb, asv, lsv, ssv, 0, 0); + int footOff = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); fbb.finish(footOff); return slice(fbb); } - private static ByteBuffer flatLayout(int encodingIdx) { - var fbb = new FlatBufferBuilder(128); - int segV = Layout.createSegmentsVector(fbb, new long[]{0}); - int off = Layout.createLayout(fbb, encodingIdx, 1L, 0, 0, segV); - Layout.finishLayoutBuffer(fbb, off); + private static MemorySegment flatLayout(int encodingIdx) { + var fbb = new FbsBuilder(128); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + int off = FbsLayout.createFbsLayout(fbb, encodingIdx, 1L, 0, 0, segV); + FbsLayout.finishFbsLayoutBuffer(fbb, off); return slice(fbb); } - private static ByteBuffer flatLayoutWithMetadata(int metadataBytes) { - var fbb = new FlatBufferBuilder(metadataBytes + 128); - int meta = Layout.createMetadataVector(fbb, new byte[metadataBytes]); - int segV = Layout.createSegmentsVector(fbb, new long[]{0}); - int off = Layout.createLayout(fbb, 0, 1L, meta, 0, segV); - Layout.finishLayoutBuffer(fbb, off); + private static MemorySegment flatLayoutWithMetadata(int metadataBytes) { + var fbb = new FbsBuilder(metadataBytes + 128); + int meta = FbsLayout.createMetadataVector(fbb, new byte[metadataBytes]); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + int off = FbsLayout.createFbsLayout(fbb, 0, 1L, meta, 0, segV); + FbsLayout.finishFbsLayoutBuffer(fbb, off); return slice(fbb); } - private static ByteBuffer nestedLayout(int depth) { - var fbb = new FlatBufferBuilder(depth * 32 + 64); - int segV = Layout.createSegmentsVector(fbb, new long[]{0}); - int current = Layout.createLayout(fbb, 0, 1L, 0, 0, segV); + private static MemorySegment nestedLayout(int depth) { + var fbb = new FbsBuilder(depth * 32 + 64); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + int current = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, segV); // Wrap `depth` times: the innermost leaf ends up at recursion depth == `depth`. for (int i = 0; i < depth; i++) { - int childV = Layout.createChildrenVector(fbb, new int[]{current}); - current = Layout.createLayout(fbb, 0, 1L, 0, childV, 0); + int childV = FbsLayout.createChildrenVector(fbb, new int[]{current}); + current = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, childV, 0); } - Layout.finishLayoutBuffer(fbb, current); + FbsLayout.finishFbsLayoutBuffer(fbb, current); return slice(fbb); } - private static ByteBuffer decimalDtype(int precision, byte scale) { - var fbb = new FlatBufferBuilder(64); - int dec = Decimal.createDecimal(fbb, precision, scale, false); - int off = io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Decimal, dec); - io.github.dfa1.vortex.fbs.DType.finishDTypeBuffer(fbb, off); + private static MemorySegment decimalDtype(int precision, byte scale) { + var fbb = new FbsBuilder(64); + int dec = FbsDecimal.createFbsDecimal(fbb, precision, scale, false); + int off = io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsDecimal, dec); + io.github.dfa1.vortex.fbs.FbsDType.finishFbsDTypeBuffer(fbb, off); return slice(fbb); } - private static ByteBuffer slice(FlatBufferBuilder fbb) { - ByteBuffer data = fbb.dataBuffer(); - return data.slice(data.position(), data.remaining()); + private static MemorySegment slice(FbsBuilder fbb) { + return fbb.dataSegment(); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java index 203a4056d..bf0bdadd4 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java @@ -13,7 +13,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -27,7 +26,7 @@ void decodeUnknownEncodingThrowsByDefault() { // Given ReadRegistry sut = ReadRegistry.empty(); ArrayNode node = new UnknownArrayNode("some.unknown", - ByteBuffer.allocate(0), new ArrayNode[0], new int[0]); + MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -42,7 +41,7 @@ void decodeKnownEncodingWithoutDecoderThrowsByDefault() { // Given — EncodingId is known but no decoder registered for it ReadRegistry sut = ReadRegistry.empty(); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, - ByteBuffer.allocate(0), new ArrayNode[0], new int[0]); + MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -57,7 +56,7 @@ void decodeKnownEncodingWithoutDecoderReturnsUnknownArrayWhenAllowed() { // Given ReadRegistry sut = ReadRegistry.builder().allowUnknown().build(); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, - ByteBuffer.allocate(0), new ArrayNode[0], new int[0]); + MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -73,7 +72,7 @@ void decodeKnownEncodingWithoutDecoderReturnsUnknownArrayWhenAllowed() { void decodeUnknownEncodingReturnsUnknownArrayWhenAllowed() { // Given ReadRegistry sut = ReadRegistry.builder().allowUnknown().build(); - ByteBuffer metadata = ByteBuffer.wrap(new byte[]{1, 2, 3}); + MemorySegment metadata = MemorySegment.ofArray(new byte[]{1, 2, 3}); MemorySegment buf = Arena.ofAuto().allocate(4); buf.set(java.lang.foreign.ValueLayout.JAVA_INT, 0, 42); ArrayNode node = new UnknownArrayNode("some.unknown", @@ -103,9 +102,9 @@ void decodeUnknownEncodingWrapsChildrenAsUnknown() { // Child uses a known id; allow-unknown still wraps it unknown because // its parent is unknown — mirrors Rust decode_foreign in vortex-array/src/serde.rs:380. ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, - ByteBuffer.allocate(0), new ArrayNode[0], new int[0]); + MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); ArrayNode parent = new UnknownArrayNode("some.unknown", - ByteBuffer.allocate(0), new ArrayNode[]{child}, new int[0]); + MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = new DecodeContext(parent, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ZipBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ZipBombSecurityTest.java index 3643e7bdc..17ad3ee42 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ZipBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ZipBombSecurityTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import static io.github.dfa1.vortex.reader.MalformedFiles.buildFooter; import static io.github.dfa1.vortex.reader.MalformedFiles.buildI64Dtype; import static io.github.dfa1.vortex.reader.MalformedFiles.buildFlatLayout; @@ -12,11 +12,11 @@ import io.github.dfa1.vortex.reader.decode.ConstantEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; -import io.github.dfa1.vortex.fbs.ArrayNode; -import io.github.dfa1.vortex.fbs.Layout; +import io.github.dfa1.vortex.fbs.FbsArrayNode; +import io.github.dfa1.vortex.fbs.FbsLayout; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -107,14 +107,14 @@ void attack2_dictLayout_inflatedCodesRowCount(@TempDir Path tmp) throws Exceptio * [Footer: arraySpecs=["vortex.constant"], layoutSpecs=["vortex.flat"], * segmentSpecs=[{offset=0, length=segLen}]] * [DType: I64 primitive] - * [Layout: flat { encoding=0, row_count=claimedRows, segments=[0] }] + * [FbsLayout: flat { encoding=0, row_count=claimedRows, segments=[0] }] * [Postscript] * [8-byte trailer] * */ private static Path buildConstantBomb(Path dir, long claimedRows) throws Exception { // ConstantEncoding stores the scalar value in buffer 0 as protobuf bytes. - byte[] protoBytes = ScalarValue.ofInt64Value(42L).encode(); + byte[] protoBytes = ProtoScalarValue.ofInt64Value(42L).encode(); byte[] seg = buildOneBufferSegment(protoBytes); ByteBuffer footerBuf = buildFooter( @@ -138,7 +138,7 @@ private static Path buildConstantBomb(Path dir, long claimedRows) throws Excepti * layoutSpecs=["vortex.flat","vortex.chunked","vortex.struct","vortex.dict"], * segmentSpecs=[{0, seg0Len}, {seg0Len, seg1Len}]] * [DType: I64 primitive] - * [Layout: dict { encoding=3, row_count=claimedRows, + * [FbsLayout: dict { encoding=3, row_count=claimedRows, * children=[values_flat(enc=0,row_count=1,seg=0), * codes_flat(enc=0,row_count=claimedRows,seg=1)] }] * [Postscript] @@ -172,17 +172,17 @@ private static Path buildDictBomb(Path dir, long claimedRows) throws Exception { * Builds: `[rawData][Array FlatBuffer (1 buffer)][4-byte LE fbLen]`. * *

The Array FlatBuffer describes one buffer at offset 0 with length `rawData.length`. - * Buffer index 0 in the ArrayNode refers to this buffer. + * Buffer index 0 in the FbsArrayNode refers to this buffer. */ private static byte[] buildOneBufferSegment(byte[] rawData) { - var fbb = new FlatBufferBuilder(128); + var fbb = new FbsBuilder(128); - // ArrayNode: encoding index 0, buffers=[0], no children, no metadata - int bufIdxVec = ArrayNode.createBuffersVector(fbb, new int[]{0}); - int nodeOff = ArrayNode.createArrayNode(fbb, 0, 0, 0, bufIdxVec, 0); + // FbsArrayNode: encoding index 0, buffers=[0], no children, no metadata + int bufIdxVec = FbsArrayNode.createBuffersVector(fbb, new int[]{0}); + int nodeOff = FbsArrayNode.createFbsArrayNode(fbb, 0, 0, 0, bufIdxVec, 0); // Array.buffers: one Buffer struct describing rawData - io.github.dfa1.vortex.fbs.Array.startBuffersVector(fbb, 1); + io.github.dfa1.vortex.fbs.FbsArray.startBuffersVector(fbb, 1); // FlatBuffers builds inline structs in reverse; struct layout (LE): // padding(u16) | alignmentExponent(u8) | compression(u8) | length(u32) fbb.prep(4, 8); @@ -192,14 +192,12 @@ private static byte[] buildOneBufferSegment(byte[] rawData) { fbb.putShort((short) 0); // padding = 0 int bufsVec = fbb.endVector(); - int arrOff = io.github.dfa1.vortex.fbs.Array.createArray(fbb, nodeOff, bufsVec); - io.github.dfa1.vortex.fbs.Array.finishArrayBuffer(fbb, arrOff); + int arrOff = io.github.dfa1.vortex.fbs.FbsArray.createFbsArray(fbb, nodeOff, bufsVec); + io.github.dfa1.vortex.fbs.FbsArray.finishFbsArrayBuffer(fbb, arrOff); // Segment = rawData + FlatBuffer bytes + 4-byte LE fbLen - ByteBuffer data = fbb.dataBuffer(); - int fbLen = data.remaining(); - byte[] fbBytes = new byte[fbLen]; - data.get(fbBytes); + byte[] fbBytes = fbb.sizedByteArray(); + int fbLen = fbBytes.length; byte[] seg = new byte[rawData.length + fbLen + 4]; System.arraycopy(rawData, 0, seg, 0, rawData.length); @@ -225,15 +223,15 @@ private static byte[] buildOneBufferSegment(byte[] rawData) { * then calls `expandDictPrimitive(..., n, arena)` which allocates `n * elemBytes`. */ private static ByteBuffer buildDictLayout(long claimedRows) { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); // Children must be built before the parent table - int vSegV = Layout.createSegmentsVector(fbb, new long[]{0}); - int valuesFlat = Layout.createLayout(fbb, 0, 1L, 0, 0, vSegV); - int cSegV = Layout.createSegmentsVector(fbb, new long[]{1}); - int codesFlat = Layout.createLayout(fbb, 0, claimedRows, 0, 0, cSegV); - int childV = Layout.createChildrenVector(fbb, new int[]{valuesFlat, codesFlat}); - int dictOff = Layout.createLayout(fbb, 3, claimedRows, 0, childV, 0); - Layout.finishLayoutBuffer(fbb, dictOff); + int vSegV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + int valuesFlat = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, vSegV); + int cSegV = FbsLayout.createSegmentsVector(fbb, new long[]{1}); + int codesFlat = FbsLayout.createFbsLayout(fbb, 0, claimedRows, 0, 0, cSegV); + int childV = FbsLayout.createChildrenVector(fbb, new int[]{valuesFlat, codesFlat}); + int dictOff = FbsLayout.createFbsLayout(fbb, 3, claimedRows, 0, childV, 0); + FbsLayout.finishFbsLayoutBuffer(fbb, dictOff); return slice(fbb); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 091cbe96c..84c26557a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -2,8 +2,8 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ALPMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; @@ -56,7 +56,7 @@ private static MemorySegment leDoubles(double... vs) { @Test void decode_nonPrimitiveDtype_throws() { // Given a Utf8 dtype on an ALP node - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(new ALPMetadata(0, 0, null).encode()), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(new ProtoALPMetadata(0, 0, null).encode()), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -85,8 +85,8 @@ void decode_f64_broadcastNoPatches_returnsConstant() { // Given a single encoded value but 4 logical rows (capacity < n) and no patches: // the decoder broadcasts it into a constant array ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - byte[] meta = new ALPMetadata(2, 0, null).encode(); // exp_e=2 -> *0.01 - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(meta), new ArrayNode[]{enc}, new int[0]); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); // exp_e=2 -> *0.01 + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); DecodeContext ctx = new DecodeContext(node, F64, 4, new MemorySegment[]{leLongs(123L)}, REGISTRY, Arena.ofAuto()); // When @@ -103,8 +103,8 @@ void decode_f64_broadcastNoPatches_returnsConstant() { void decode_f32_broadcastNoPatches_returnsConstant() { // Given single value, 3 rows, no patches ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - byte[] meta = new ALPMetadata(1, 0, null).encode(); // exp_e=1 -> *0.1 - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(meta), new ArrayNode[]{enc}, new int[0]); + byte[] meta = new ProtoALPMetadata(1, 0, null).encode(); // exp_e=1 -> *0.1 + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); DecodeContext ctx = new DecodeContext(node, F32, 3, new MemorySegment[]{leInts(25)}, REGISTRY, Arena.ofAuto()); // When @@ -121,13 +121,13 @@ void decode_f32_broadcastNoPatches_returnsConstant() { void decode_f64_patches_withU8Indices() { // Given patches whose index child uses U8 storage — exercises the U8 arm of // readUnsigned (the encoder always emits U32 indices) - PatchesMetadata pm = new PatchesMetadata(1L, 0L, io.github.dfa1.vortex.proto.PType.U8, null, null, null); - byte[] meta = new ALPMetadata(2, 0, pm).encode(); // *0.01 + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, io.github.dfa1.vortex.proto.ProtoPType.U8, null, null, null); + byte[] meta = new ProtoALPMetadata(2, 0, pm).encode(); // *0.01 ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode idx = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode val = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(meta), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc, idx, val}, new int[0]); MemorySegment idxSeg = MemorySegment.ofArray(new byte[]{1}); // patch row 1 @@ -146,13 +146,13 @@ void decode_f64_patches_withU8Indices() { @Test void decode_patches_nonUnsignedIndexPtype_throws() { // Given a signed (I32) patch-index ptype — readUnsigned rejects it - PatchesMetadata pm = new PatchesMetadata(1L, 0L, io.github.dfa1.vortex.proto.PType.I32, null, null, null); - byte[] meta = new ALPMetadata(2, 0, pm).encode(); + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, io.github.dfa1.vortex.proto.ProtoPType.I32, null, null, null); + byte[] meta = new ProtoALPMetadata(2, 0, pm).encode(); ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode idx = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode val = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(meta), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc, idx, val}, new int[0]); MemorySegment[] segs = {leLongs(100L, 0L), leInts(1), leDoubles(9.0)}; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java index f39cd6094..6a51314a2 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java @@ -5,15 +5,13 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.TestSegments; import io.github.dfa1.vortex.encoding.TimeUnit; -import io.github.dfa1.vortex.proto.DateTimePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDateTimePartsMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.LongArray; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -25,24 +23,21 @@ class DateTimePartsEncodingDecoderTest { private static final long SECONDS_PER_DAY = 86_400L; - private static ByteBuffer i64Meta() { - return ByteBuffer.wrap(new DateTimePartsMetadata( - io.github.dfa1.vortex.proto.PType.I64, - io.github.dfa1.vortex.proto.PType.I64, - io.github.dfa1.vortex.proto.PType.I64).encode()); + private static MemorySegment i64Meta() { + return MemorySegment.ofArray(new ProtoDateTimePartsMetadata( + io.github.dfa1.vortex.proto.ProtoPType.I64, + io.github.dfa1.vortex.proto.ProtoPType.I64, + io.github.dfa1.vortex.proto.ProtoPType.I64).encode()); } private static DType timestampDType(TimeUnit unit, boolean nullable) { - ByteBuffer meta = ByteBuffer.allocate(3).order(ByteOrder.LITTLE_ENDIAN); - meta.put((byte) unit.ordinal()); - meta.putShort((short) 0); - meta.flip(); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) unit.ordinal(), 0, 0}); return new DType.Extension("vortex.timestamp", new DType.Primitive(PType.I64, nullable), meta, nullable); } /// Builds a context with three I64 part-children backed by the given segments. - private static DecodeContext ctx(ByteBuffer meta, DType dtype, long n, + private static DecodeContext ctx(MemorySegment meta, DType dtype, long n, MemorySegment days, MemorySegment seconds, MemorySegment subseconds) { ArrayNode d = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode s = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java index f077a4b47..e71dc5fcd 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java @@ -13,7 +13,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -26,11 +25,11 @@ class DecimalEncodingDecoderTest { private static Array decode(int valuesType, int rowCount, int bufferBytes) { // Encode values_type explicitly (field 1, varint) so even the proto3 default (0) is // present on the wire — DecimalMetadata(0).encode() would omit it and read as empty. - ByteBuffer meta = ByteBuffer.wrap(new byte[]{0x08, (byte) valuesType}); + MemorySegment meta = MemorySegment.ofArray(new byte[]{0x08, (byte) valuesType}); return decode(meta, rowCount, bufferBytes); } - private static Array decode(ByteBuffer meta, int rowCount, int bufferBytes) { + private static Array decode(MemorySegment meta, int rowCount, int bufferBytes) { MemorySegment buf = MemorySegment.ofArray(new byte[bufferBytes]); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DECIMAL, meta, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DECIMAL, rowCount, @@ -80,7 +79,7 @@ void bufferTooSmall_throws() { @Test void missingMetadata_throws() { // When / Then — null metadata - assertThatThrownBy(() -> decode((ByteBuffer) null, 1, 8)) + assertThatThrownBy(() -> decode((MemorySegment) null, 1, 8)) .isInstanceOf(VortexException.class) .hasMessageContaining("missing metadata"); } @@ -88,7 +87,7 @@ void missingMetadata_throws() { @Test void emptyMetadata_throws() { // When / Then — present but zero remaining - assertThatThrownBy(() -> decode(ByteBuffer.allocate(0), 1, 8)) + assertThatThrownBy(() -> decode(MemorySegment.ofArray(new byte[0]), 1, 8)) .isInstanceOf(VortexException.class) .hasMessageContaining("missing metadata"); } @@ -96,7 +95,7 @@ void emptyMetadata_throws() { @Test void invalidMetadata_throws() { // Given — a truncated varint that proto decode rejects - ByteBuffer bad = ByteBuffer.wrap(new byte[]{0x08, (byte) 0x80}); + MemorySegment bad = MemorySegment.ofArray(new byte[]{0x08, (byte) 0x80}); // When / Then assertThatThrownBy(() -> decode(bad, 1, 8)) diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java index 688b8b73d..a66347726 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.encoding.TestSegments; -import io.github.dfa1.vortex.proto.DeltaMetadata; +import io.github.dfa1.vortex.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LongArray; @@ -15,7 +15,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; @@ -57,7 +56,7 @@ void decode_constantChildren_broadcastsAcrossChunk() { // means every decoded row is zero. PType ptype = PType.I64; long deltasLen = FL_CHUNK_SIZE; - ByteBuffer meta = ByteBuffer.wrap(new DeltaMetadata(deltasLen, 0).encode()); + MemorySegment meta = MemorySegment.ofArray(new ProtoDeltaMetadata(deltasLen, 0).encode()); ArrayNode bases = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode deltas = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); @@ -82,7 +81,7 @@ void decode_constantBases_nonZeroOffsetAndBase() { // Given a constant base of 5 broadcast across all 16 lanes with zero deltas: // every row decodes to the base value 5. Reads from an offset into the chunk. PType ptype = PType.I64; - ByteBuffer meta = ByteBuffer.wrap(new DeltaMetadata(FL_CHUNK_SIZE, 0).encode()); + MemorySegment meta = MemorySegment.ofArray(new ProtoDeltaMetadata(FL_CHUNK_SIZE, 0).encode()); ArrayNode bases = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode deltas = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index 939252916..a752d700c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -5,8 +5,8 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.DictMetadata; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoDictMetadata; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -25,7 +25,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.stream.Stream; @@ -150,7 +149,7 @@ void missingMetadata_throws() { @Test void emptyMetadata_throws() { // Given — metadata present but with zero remaining bytes (exercises !hasRemaining) - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, ByteBuffer.allocate(0), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.I32, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -164,7 +163,7 @@ void emptyMetadata_throws() { @Test void malformedProtoMetadata_throws() { // Given — >1 byte (routes to proto path) but a truncated varint that proto decode rejects - ByteBuffer meta = ByteBuffer.wrap(new byte[]{0x08, (byte) 0x80}); + MemorySegment meta = MemorySegment.ofArray(new byte[]{0x08, (byte) 0x80}); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.I32, @@ -298,7 +297,7 @@ void legacyLayout_decodesStringsByCode() { MemorySegment offsets = TestSegments.leLongs(0, 2, 5); MemorySegment codes = u8Codes(1, 0, 1); - ByteBuffer meta = ByteBuffer.wrap(new byte[]{(byte) PType.U8.ordinal()}); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) PType.U8.ordinal()}); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[0], new int[]{0, 1, 2}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 3, new MemorySegment[]{bytes, offsets, codes}, REGISTRY, Arena.ofAuto()); @@ -321,9 +320,9 @@ void protoLayout_decodesStringsByCode() { MemorySegment codes = u8Codes(0, 1, 0); MemorySegment[] segs = {codes, bytes, offsets}; - ByteBuffer dictMeta = ByteBuffer.wrap( - new DictMetadata(2, protoPType(PType.U8), null, null).encode()); - ByteBuffer varBinMeta = ByteBuffer.wrap(new VarBinMetadata(protoPType(PType.I64)).encode()); + MemorySegment dictMeta = MemorySegment.ofArray( + new ProtoDictMetadata(2, protoPType(PType.U8), null, null).encode()); + MemorySegment varBinMeta = MemorySegment.ofArray(new ProtoVarBinMetadata(protoPType(PType.I64)).encode()); ArrayNode codesNode = primitiveNode(0); ArrayNode offsetsNode = primitiveNode(2); @@ -360,7 +359,7 @@ void legacyLayout_missingMetadata_throws() { @Test void protoLayout_malformedMetadata_throws() { // Given — children present, metadata is an invalid (truncated varint) proto blob - ByteBuffer meta = ByteBuffer.wrap(new byte[]{0x08, (byte) 0x80}); + MemorySegment meta = MemorySegment.ofArray(new byte[]{0x08, (byte) 0x80}); ArrayNode child = primitiveNode(0); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{child, child}, new int[]{}); @@ -390,7 +389,7 @@ void protoLayout_missingMetadata_throws() { @Test void legacyLayout_emptyMetadata_throws() { // Given — no children and zero-remaining metadata (exercises !hasRemaining) - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, ByteBuffer.allocate(0), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -405,7 +404,7 @@ void legacyLayout_emptyMetadata_throws() { void protoLayout_emptyMetadata_throws() { // Given — children present, zero-remaining metadata ArrayNode child = primitiveNode(0); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, ByteBuffer.allocate(0), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child, child}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[]{u8Codes(0)}, REGISTRY, Arena.ofAuto()); @@ -441,7 +440,7 @@ private static Array decodeProto(DType dtype, PType codePType, long[] codes, lon private static Array decodeProtoSegments(DType dtype, PType codePType, MemorySegment codes, MemorySegment values, int valuesLen, int rowCount) { - ByteBuffer meta = ByteBuffer.wrap(new DictMetadata(valuesLen, protoPType(codePType), null, null).encode()); + MemorySegment meta = MemorySegment.ofArray(new ProtoDictMetadata(valuesLen, protoPType(codePType), null, null).encode()); MemorySegment[] segs = {codes, values}; ArrayNode dictNode = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); @@ -451,7 +450,7 @@ private static Array decodeProtoSegments(DType dtype, PType codePType, MemorySeg private static Array decodeLegacy(DType dtype, PType codePType, MemorySegment values, MemorySegment codes, int rowCount) { - ByteBuffer meta = ByteBuffer.wrap(new byte[]{(byte) codePType.ordinal()}); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) codePType.ordinal()}); MemorySegment[] segs = {values, codes}; ArrayNode dictNode = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); @@ -463,8 +462,8 @@ private static ArrayNode primitiveNode(int bufferIndex) { return ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{bufferIndex}); } - private static io.github.dfa1.vortex.proto.PType protoPType(PType core) { - return io.github.dfa1.vortex.proto.PType.valueOf(core.name()); + private static io.github.dfa1.vortex.proto.ProtoPType protoPType(PType core) { + return io.github.dfa1.vortex.proto.ProtoPType.valueOf(core.name()); } // ── segment builders (little-endian) ─────────────────────────────────────── diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java index 2edf70fab..00459fa82 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java @@ -8,14 +8,13 @@ import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.PatchedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchedMetadata; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -25,8 +24,8 @@ class PatchedEncodingDecoderTest { private static final PatchedEncodingDecoder SUT = new PatchedEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); - private static ByteBuffer patchedMeta(int nPatches, int nLanes, int offset) { - return ByteBuffer.wrap(new PatchedMetadata(nPatches, nLanes, offset).encode()); + private static MemorySegment patchedMeta(int nPatches, int nLanes, int offset) { + return MemorySegment.ofArray(new ProtoPatchedMetadata(nPatches, nLanes, offset).encode()); } @@ -45,7 +44,7 @@ private static Array decode(DType dtype, int n, MemorySegment patchIndices, MemorySegment patchValues, int nLanes) { int nPatches = (int) (patchIndices.byteSize() / 2); - ByteBuffer meta = patchedMeta(nPatches, nLanes, 0); + MemorySegment meta = patchedMeta(nPatches, nLanes, 0); MemorySegment[] segments = {inner, laneOffsets, patchIndices, patchValues}; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java index 3dd07564e..082ab6561 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java @@ -9,9 +9,9 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.PcoChunkInfo; -import io.github.dfa1.vortex.proto.PcoMetadata; -import io.github.dfa1.vortex.proto.PcoPageInfo; +import io.github.dfa1.vortex.proto.ProtoPcoChunkInfo; +import io.github.dfa1.vortex.proto.ProtoPcoMetadata; +import io.github.dfa1.vortex.proto.ProtoPcoPageInfo; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -22,7 +22,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.util.Random; import java.util.stream.Stream; @@ -34,18 +33,18 @@ class PcoEncodingDecoderTest { private static final PcoEncodingDecoder SUT = new PcoEncodingDecoder(); - private static ByteBuffer validMetaBuffer() { - PcoMetadata meta = new PcoMetadata(new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, java.util.List.of()); - return ByteBuffer.wrap(meta.encode()); + private static MemorySegment validMetaBuffer() { + ProtoPcoMetadata meta = new ProtoPcoMetadata(new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, java.util.List.of()); + return MemorySegment.ofArray(meta.encode()); } - private static DecodeContext ctxWith(ByteBuffer meta, DType dtype, long rowCount, MemorySegment[] buffers) { + private static DecodeContext ctxWith(MemorySegment meta, DType dtype, long rowCount, MemorySegment[] buffers) { ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PCO, meta, new ArrayNode[0], bufferIndices(buffers.length)); return new DecodeContext(node, dtype, rowCount, buffers, ReadRegistry.empty(), Arena.ofAuto()); } - private static DecodeContext ctxWithValidity(ByteBuffer meta, DType dtype, long rowCount, + private static DecodeContext ctxWithValidity(MemorySegment meta, DType dtype, long rowCount, MemorySegment validityBuf, MemorySegment[] pcoBuffers) { MemorySegment[] allBuffers = new MemorySegment[1 + pcoBuffers.length]; allBuffers[0] = validityBuf; @@ -81,11 +80,11 @@ private static MemorySegment segmentOf(byte... bytes) { return seg; } - private static ByteBuffer metaWithOneChunk(int nValues) { - PcoMetadata meta = new PcoMetadata( + private static MemorySegment metaWithOneChunk(int nValues) { + ProtoPcoMetadata meta = new ProtoPcoMetadata( new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, - java.util.List.of(new PcoChunkInfo(java.util.List.of(new PcoPageInfo(nValues))))); - return ByteBuffer.wrap(meta.encode()); + java.util.List.of(new ProtoPcoChunkInfo(java.util.List.of(new ProtoPcoPageInfo(nValues))))); + return MemorySegment.ofArray(meta.encode()); } private static MemorySegment chunkMetaConsecutive(int order) { @@ -173,13 +172,13 @@ void decode_nullMetadata_throwsMissingMeta() { DecodeContext ctx = ctxWith(null, DType.I64, 0, new MemorySegment[0]); assertThatThrownBy(() -> SUT.decode(ctx)) .isInstanceOf(VortexException.class) - .hasMessageContaining("missing PcoMetadata"); + .hasMessageContaining("missing ProtoPcoMetadata"); } @Test void decode_invalidHeaderVersion_throwsUnsupported() { - PcoMetadata meta = new PcoMetadata(new byte[]{0x03, 0x00}, java.util.List.of()); - DecodeContext ctx = ctxWith(ByteBuffer.wrap(meta.encode()), + ProtoPcoMetadata meta = new ProtoPcoMetadata(new byte[]{0x03, 0x00}, java.util.List.of()); + DecodeContext ctx = ctxWith(MemorySegment.ofArray(meta.encode()), DType.I64, 0, new MemorySegment[0]); assertThatThrownBy(() -> SUT.decode(ctx)) .isInstanceOf(VortexException.class) @@ -232,10 +231,10 @@ void decode_consecutiveDelta_order2_twoValues_decodes() { @Test void decode_multiPage_singleChunk_decodes() { - PcoMetadata meta = new PcoMetadata( + ProtoPcoMetadata meta = new ProtoPcoMetadata( new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, - java.util.List.of(new PcoChunkInfo(java.util.List.of(new PcoPageInfo(1), new PcoPageInfo(1))))); - DecodeContext ctx = ctxWith(ByteBuffer.wrap(meta.encode()), DType.U64, 2, + java.util.List.of(new ProtoPcoChunkInfo(java.util.List.of(new ProtoPcoPageInfo(1), new ProtoPcoPageInfo(1))))); + DecodeContext ctx = ctxWith(MemorySegment.ofArray(meta.encode()), DType.U64, 2, new MemorySegment[]{chunkMetaConsecutive(1), pageWithMoments(10L), pageWithMoments(20L)}); var result = SUT.decode(ctx); assertThat(result.length()).isEqualTo(2); @@ -246,12 +245,12 @@ void decode_multiPage_singleChunk_decodes() { @Test void decode_multiChunk_decodes() { // Buffer layout: all chunk metas first, then all pages (matches Rust vortex PcoArray). - PcoMetadata meta = new PcoMetadata( + ProtoPcoMetadata meta = new ProtoPcoMetadata( new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, java.util.List.of( - new PcoChunkInfo(java.util.List.of(new PcoPageInfo(1))), - new PcoChunkInfo(java.util.List.of(new PcoPageInfo(1))))); - DecodeContext ctx = ctxWith(ByteBuffer.wrap(meta.encode()), DType.U64, 2, + new ProtoPcoChunkInfo(java.util.List.of(new ProtoPcoPageInfo(1))), + new ProtoPcoChunkInfo(java.util.List.of(new ProtoPcoPageInfo(1))))); + DecodeContext ctx = ctxWith(MemorySegment.ofArray(meta.encode()), DType.U64, 2, new MemorySegment[]{chunkMetaConsecutive(1), chunkMetaConsecutive(1), pageWithMoments(100L), pageWithMoments(200L)}); var result = SUT.decode(ctx); @@ -266,12 +265,12 @@ class DecodeNullable { @Test void decode_nullable_someNulls_scattersCorrectly() { - PcoMetadata meta = new PcoMetadata( + ProtoPcoMetadata meta = new ProtoPcoMetadata( new byte[]{PcoEncodingDecoder.PCO_FORMAT_MAJOR, PcoEncodingDecoder.PCO_FORMAT_MINOR}, - java.util.List.of(new PcoChunkInfo(java.util.List.of(new PcoPageInfo(1), new PcoPageInfo(1))))); + java.util.List.of(new ProtoPcoChunkInfo(java.util.List.of(new ProtoPcoPageInfo(1), new ProtoPcoPageInfo(1))))); MemorySegment validityBuf = segmentOf((byte) 0x05); DecodeContext ctx = ctxWithValidity( - ByteBuffer.wrap(meta.encode()), new DType.Primitive(PType.U64, true), 3, validityBuf, + MemorySegment.ofArray(meta.encode()), new DType.Primitive(PType.U64, true), 3, validityBuf, new MemorySegment[]{chunkMetaConsecutive(1), pageWithMoments(100L), pageWithMoments(200L)}); var result = SUT.decode(ctx); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java index 1dbf10ce7..36f0c77d2 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.TestSegments; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VarBinArray; @@ -11,7 +11,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import static org.assertj.core.api.Assertions.assertThat; @@ -21,11 +20,11 @@ class VarBinEncodingDecoderTest { private static final VarBinEncodingDecoder SUT = new VarBinEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); - private static ByteBuffer i32OffsetsMeta() { - return ByteBuffer.wrap(new VarBinMetadata(io.github.dfa1.vortex.proto.PType.I32).encode()); + private static MemorySegment i32OffsetsMeta() { + return MemorySegment.ofArray(new ProtoVarBinMetadata(io.github.dfa1.vortex.proto.ProtoPType.I32).encode()); } - private static DecodeContext ctx(ByteBuffer meta, MemorySegment bytes, MemorySegment offsets, long n) { + private static DecodeContext ctx(MemorySegment meta, MemorySegment bytes, MemorySegment offsets, long n) { // children[0] = offsets (primitive, segment index 1); bufferIndices[0] -> bytes (index 0) ArrayNode offsetsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode varbinNode = ArrayNode.of(EncodingId.VORTEX_VARBIN, meta, new ArrayNode[]{offsetsNode}, new int[]{0}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java index 8dce5eed8..4e9e18958 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java @@ -9,14 +9,13 @@ import io.github.dfa1.vortex.reader.array.VariantArray; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.Primitive; -import io.github.dfa1.vortex.proto.VariantMetadata; +import io.github.dfa1.vortex.proto.ProtoPrimitive; +import io.github.dfa1.vortex.proto.ProtoVariantMetadata; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -29,8 +28,8 @@ class VariantEncodingDecoderTest { private static final VariantEncodingDecoder SUT = new VariantEncodingDecoder(); - private static ByteBuffer variantMetaWithShredded(io.github.dfa1.vortex.proto.DType shredded) { - return ByteBuffer.wrap(new VariantMetadata(shredded).encode()); + private static MemorySegment variantMetaWithShredded(io.github.dfa1.vortex.proto.ProtoDType shredded) { + return MemorySegment.ofArray(new ProtoVariantMetadata(shredded).encode()); } private static ArrayNode primitiveChildNode(int segIdx) { @@ -68,9 +67,9 @@ void decode_withoutShredded_returnsCoreStorageOnly() { @Test void decode_withShredded_decodesSecondChild() { // Given - io.github.dfa1.vortex.proto.DType shreddedProto = io.github.dfa1.vortex.proto.DType.ofPrimitive( - new Primitive(io.github.dfa1.vortex.proto.PType.I32, false)); - ByteBuffer meta = variantMetaWithShredded(shreddedProto); + io.github.dfa1.vortex.proto.ProtoDType shreddedProto = io.github.dfa1.vortex.proto.ProtoDType.ofPrimitive( + new ProtoPrimitive(io.github.dfa1.vortex.proto.ProtoPType.I32, false)); + MemorySegment meta = variantMetaWithShredded(shreddedProto); ArrayNode coreNode = nullChildNode(); ArrayNode shreddedNode = primitiveChildNode(0); @@ -97,7 +96,7 @@ void decode_withShredded_decodesSecondChild() { void decode_emptyMetadata_noShredded() { // Given ArrayNode coreNode = nullChildNode(); - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, ByteBuffer.allocate(0), + ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{coreNode}, new int[]{}); ReadRegistry registry = TestRegistry.ofDecoders(SUT, new NullEncodingDecoder()); @@ -152,15 +151,15 @@ void decode_wrongChildCount_throws() { @Nested class DtypeFromProto { - private static io.github.dfa1.vortex.proto.DType prim(io.github.dfa1.vortex.proto.PType pt, boolean nullable) { - return io.github.dfa1.vortex.proto.DType.ofPrimitive(new Primitive(pt, nullable)); + private static io.github.dfa1.vortex.proto.ProtoDType prim(io.github.dfa1.vortex.proto.ProtoPType pt, boolean nullable) { + return io.github.dfa1.vortex.proto.ProtoDType.ofPrimitive(new ProtoPrimitive(pt, nullable)); } @Test void nullType() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofNull(new io.github.dfa1.vortex.proto.Null())); + io.github.dfa1.vortex.proto.ProtoDType.ofNull(new io.github.dfa1.vortex.proto.ProtoNull())); // Then null is always nullable assertThat(result).isEqualTo(new DType.Null(true)); @@ -170,7 +169,7 @@ void nullType() { void bool() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofBool(new io.github.dfa1.vortex.proto.Bool(true))); + io.github.dfa1.vortex.proto.ProtoDType.ofBool(new io.github.dfa1.vortex.proto.ProtoBool(true))); // Then assertThat(result).isEqualTo(new DType.Bool(true)); @@ -179,7 +178,7 @@ void bool() { @Test void primitive() { // Given / When - DType result = VariantEncodingDecoder.dtypeFromProto(prim(io.github.dfa1.vortex.proto.PType.I64, false)); + DType result = VariantEncodingDecoder.dtypeFromProto(prim(io.github.dfa1.vortex.proto.ProtoPType.I64, false)); // Then assertThat(result).isEqualTo(DType.I64); @@ -189,7 +188,7 @@ void primitive() { void decimal() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofDecimal(new io.github.dfa1.vortex.proto.Decimal(10, 2, false))); + io.github.dfa1.vortex.proto.ProtoDType.ofDecimal(new io.github.dfa1.vortex.proto.ProtoDecimal(10, 2, false))); // Then precision/scale narrow to byte assertThat(result).isEqualTo(new DType.Decimal((byte) 10, (byte) 2, false)); @@ -199,7 +198,7 @@ void decimal() { void utf8() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofUtf8(new io.github.dfa1.vortex.proto.Utf8(true))); + io.github.dfa1.vortex.proto.ProtoDType.ofUtf8(new io.github.dfa1.vortex.proto.ProtoUtf8(true))); // Then assertThat(result).isEqualTo(new DType.Utf8(true)); @@ -209,7 +208,7 @@ void utf8() { void binary() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofBinary(new io.github.dfa1.vortex.proto.Binary(false))); + io.github.dfa1.vortex.proto.ProtoDType.ofBinary(new io.github.dfa1.vortex.proto.ProtoBinary(false))); // Then assertThat(result).isEqualTo(DType.BINARY); @@ -218,10 +217,10 @@ void binary() { @Test void struct() { // Given a two-field struct with mixed child types - var proto = io.github.dfa1.vortex.proto.DType.ofStruct(new io.github.dfa1.vortex.proto.Struct( + var proto = io.github.dfa1.vortex.proto.ProtoDType.ofStruct(new io.github.dfa1.vortex.proto.ProtoStruct( List.of("a", "b"), - List.of(prim(io.github.dfa1.vortex.proto.PType.I32, false), - io.github.dfa1.vortex.proto.DType.ofUtf8(new io.github.dfa1.vortex.proto.Utf8(true))), + List.of(prim(io.github.dfa1.vortex.proto.ProtoPType.I32, false), + io.github.dfa1.vortex.proto.ProtoDType.ofUtf8(new io.github.dfa1.vortex.proto.ProtoUtf8(true))), false)); // When children are translated recursively @@ -238,8 +237,8 @@ void struct() { void list() { // Given / When element type is translated recursively DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofList(new io.github.dfa1.vortex.proto.List( - prim(io.github.dfa1.vortex.proto.PType.I32, false), true))); + io.github.dfa1.vortex.proto.ProtoDType.ofList(new io.github.dfa1.vortex.proto.ProtoList( + prim(io.github.dfa1.vortex.proto.ProtoPType.I32, false), true))); // Then assertThat(result).isEqualTo(new DType.List(DType.I32, true)); @@ -249,8 +248,8 @@ void list() { void fixedSizeList() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofFixedSizeList(new io.github.dfa1.vortex.proto.FixedSizeList( - prim(io.github.dfa1.vortex.proto.PType.F64, false), 4, false))); + io.github.dfa1.vortex.proto.ProtoDType.ofFixedSizeList(new io.github.dfa1.vortex.proto.ProtoFixedSizeList( + prim(io.github.dfa1.vortex.proto.ProtoPType.F64, false), 4, false))); // Then size is carried through assertThat(result).isEqualTo( @@ -260,8 +259,8 @@ void fixedSizeList() { @Test void extension_withMetadata() { // Given an extension with non-null metadata bytes - var proto = io.github.dfa1.vortex.proto.DType.ofExtension(new io.github.dfa1.vortex.proto.Extension( - "ip.address", prim(io.github.dfa1.vortex.proto.PType.I32, false), new byte[]{1, 2, 3})); + var proto = io.github.dfa1.vortex.proto.ProtoDType.ofExtension(new io.github.dfa1.vortex.proto.ProtoExtension( + "ip.address", prim(io.github.dfa1.vortex.proto.ProtoPType.I32, false), new byte[]{1, 2, 3})); // When DType result = VariantEncodingDecoder.dtypeFromProto(proto); @@ -271,27 +270,27 @@ void extension_withMetadata() { DType.Extension ext = (DType.Extension) result; assertThat(ext.extensionId()).isEqualTo("ip.address"); assertThat(ext.storageDType()).isEqualTo(DType.I32); - assertThat(ext.metadata().remaining()).isEqualTo(3); + assertThat(ext.metadata().byteSize()).isEqualTo(3); } @Test void extension_nullMetadata_becomesEmptyBuffer() { // Given null metadata — must not NPE, maps to an empty read-only buffer - var proto = io.github.dfa1.vortex.proto.DType.ofExtension(new io.github.dfa1.vortex.proto.Extension( - "uuid", prim(io.github.dfa1.vortex.proto.PType.I64, false), null)); + var proto = io.github.dfa1.vortex.proto.ProtoDType.ofExtension(new io.github.dfa1.vortex.proto.ProtoExtension( + "uuid", prim(io.github.dfa1.vortex.proto.ProtoPType.I64, false), null)); // When DType.Extension result = (DType.Extension) VariantEncodingDecoder.dtypeFromProto(proto); // Then - assertThat(result.metadata().remaining()).isZero(); + assertThat(result.metadata().byteSize()).isZero(); } @Test void variant() { // Given / When DType result = VariantEncodingDecoder.dtypeFromProto( - io.github.dfa1.vortex.proto.DType.ofVariant(new io.github.dfa1.vortex.proto.Variant(false))); + io.github.dfa1.vortex.proto.ProtoDType.ofVariant(new io.github.dfa1.vortex.proto.ProtoVariant(false))); // Then assertThat(result).isEqualTo(DType.VARIANT); @@ -300,7 +299,7 @@ void variant() { @Test void noFieldSet_throws() { // Given a proto DType with no oneof arm populated - var empty = new io.github.dfa1.vortex.proto.DType( + var empty = new io.github.dfa1.vortex.proto.ProtoDType( null, null, null, null, null, null, null, null, null, null, null, null); // When / Then diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java index 10eb27a66..67ba31ec1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java @@ -40,8 +40,8 @@ void dtype_isI32StorageWithDaysMetadata() { assertThat(dtype.extensionId()).isEqualTo("vortex.date"); assertThat(dtype.storageDType()).isEqualTo(DType.I32); assertThat(dtype.metadata()).isNotNull(); - assertThat(dtype.metadata().remaining()).isEqualTo(1); - assertThat(dtype.metadata().duplicate().get()).isEqualTo((byte) 4); // TimeUnit.Days ordinal + assertThat(dtype.metadata().byteSize()).isEqualTo(1); + assertThat(dtype.metadata().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) 4); // TimeUnit.Days ordinal } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java index 623a11373..38cb5ff2a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.time.Instant; import static io.github.dfa1.vortex.reader.array.TestArrays.bools; @@ -70,7 +70,7 @@ class ReadUnit { @Test void readsTagByte() { // Given — tag 3 = Seconds (ordinal) - DType.Extension ext = ext(ByteBuffer.wrap(new byte[]{3})); + DType.Extension ext = ext(java.lang.foreign.MemorySegment.ofArray(new byte[]{3})); // When / Then assertThat(ExtensionStorage.readUnit(ext)).isEqualTo(TimeUnit.Seconds); @@ -84,11 +84,11 @@ void nullMetadata_throws() { @Test void emptyMetadata_throws() { - assertThatThrownBy(() -> ExtensionStorage.readUnit(ext(ByteBuffer.allocate(0)))) + assertThatThrownBy(() -> ExtensionStorage.readUnit(ext(MemorySegment.ofArray(new byte[0])))) .isInstanceOf(VortexException.class).hasMessageContaining("missing TimeUnit"); } - private DType.Extension ext(ByteBuffer meta) { + private DType.Extension ext(java.lang.foreign.MemorySegment meta) { return new DType.Extension("vortex.timestamp", DType.I64, meta, false); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java index 1acbaed78..e1e8cf22b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java @@ -6,11 +6,11 @@ import io.github.dfa1.vortex.reader.array.MaterializedIntArray; import io.github.dfa1.vortex.reader.array.MaterializedLongArray; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT; + import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; /// Shared fixtures for the per-extension decoder test classes. @@ -23,24 +23,20 @@ final class ExtensionTestSupport { private ExtensionTestSupport() { } - static DType.Extension ext(String id, DType storage, ByteBuffer meta) { + static DType.Extension ext(String id, DType storage, MemorySegment meta) { return new DType.Extension(id, storage, meta, false); } - static ByteBuffer unitByte(byte tag) { - ByteBuffer meta = ByteBuffer.allocate(1); - meta.put(0, tag); - return meta; + static MemorySegment unitByte(byte tag) { + return MemorySegment.ofArray(new byte[]{tag}); } - static ByteBuffer tzMeta(byte unitTag, String tz) { + static MemorySegment tzMeta(byte unitTag, String tz) { byte[] tzBytes = tz == null ? new byte[0] : tz.getBytes(StandardCharsets.UTF_8); - ByteBuffer meta = ByteBuffer.allocate(3 + tzBytes.length).order(ByteOrder.LITTLE_ENDIAN); - meta.put(0, unitTag); - meta.putShort(1, (short) tzBytes.length); - for (int k = 0; k < tzBytes.length; k++) { - meta.put(3 + k, tzBytes[k]); - } + MemorySegment meta = MemorySegment.ofArray(new byte[3 + tzBytes.length]); + meta.set(ValueLayout.JAVA_BYTE, 0, unitTag); + meta.set(LE_SHORT, 1, (short) tzBytes.length); + MemorySegment.copy(tzBytes, 0, meta, ValueLayout.JAVA_BYTE, 3, tzBytes.length); return meta; } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java index 11b4af133..c30a9b67a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java @@ -45,7 +45,7 @@ void dtype_defaultsToMillisecondsI32() { // Then — storage width is I32 for s/ms, metadata byte 0 = TimeUnit.Milliseconds.ordinal() assertThat(dtype.storageDType()).isEqualTo(new DType.Primitive(PType.I32, true)); - assertThat(dtype.metadata().get(0)).isEqualTo((byte) TimeUnit.Milliseconds.ordinal()); + assertThat(dtype.metadata().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) TimeUnit.Milliseconds.ordinal()); } @Test @@ -55,7 +55,7 @@ void dtype_nanosecondsForceI64() { // Then assertThat(dtype.storageDType()).isEqualTo(DType.I64); - assertThat(dtype.metadata().get(0)).isEqualTo((byte) TimeUnit.Nanoseconds.ordinal()); + assertThat(dtype.metadata().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) TimeUnit.Nanoseconds.ordinal()); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java index 604d75619..28856790f 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java @@ -15,8 +15,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; @@ -47,8 +45,8 @@ void dtype_defaultIsMsUtcless() { // Then — byte 0 = ms ordinal, bytes 1..3 = 0 (tz_len = 0) assertThat(dtype.storageDType()).isEqualTo(DType.I64); - assertThat(dtype.metadata().get(0)).isEqualTo((byte) TimeUnit.Milliseconds.ordinal()); - assertThat(dtype.metadata().getShort(1)).isEqualTo((short) 0); + assertThat(dtype.metadata().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) TimeUnit.Milliseconds.ordinal()); + assertThat(dtype.metadata().get(io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT, 1)).isEqualTo((short) 0); } @Test @@ -57,7 +55,7 @@ void dtype_withTimezoneEncodesIanaName() { DType.Extension dtype = sut.dtype(TimeUnit.Microseconds, ZoneId.of("Europe/Paris"), false); // Then — header tz_len matches the UTF-8 length; the actual bytes follow - int tzLen = Short.toUnsignedInt(dtype.metadata().getShort(1)); + int tzLen = Short.toUnsignedInt(dtype.metadata().get(io.github.dfa1.vortex.encoding.PTypeIO.LE_SHORT, 1)); assertThat(tzLen).isEqualTo("Europe/Paris".getBytes().length); assertThat(sut.timezone(dtype)).contains(ZoneId.of("Europe/Paris")); } @@ -149,12 +147,7 @@ void decodeAll_maskedArray_yieldsNullAtInvalidPositions() { @Test void timezone_truncatedMetadata_throws() { // Given — declared tz_len longer than buffer can carry - ByteBuffer meta = ByteBuffer.allocate(6).order(ByteOrder.LITTLE_ENDIAN); - meta.put(0, (byte) 2); - meta.putShort(1, (short) 5); - meta.put(3, (byte) 'U'); - meta.put(4, (byte) 'T'); - meta.put(5, (byte) 'C'); + MemorySegment meta = MemorySegment.ofArray(new byte[]{2, 5, 0, (byte) 0x55, (byte) 0x54, (byte) 0x43}); DType.Extension truncated = ext("vortex.timestamp", I64, meta); // When / Then diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoderTest.java index 9565f60a7..15f373c90 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoderTest.java @@ -42,7 +42,7 @@ void dtype_isFixedSizeListOf16U8() { DType.Primitive u8 = DType.U8; assertThat(dtype.storageDType()).isEqualTo(new DType.FixedSizeList(u8, 16, true)); assertThat(dtype.metadata()).isNotNull(); - assertThat(dtype.metadata().remaining()).isZero(); + assertThat(dtype.metadata().byteSize()).isZero(); } @Test diff --git a/writer/pom.xml b/writer/pom.xml index fa8ac88e6..2ddb17f36 100644 --- a/writer/pom.xml +++ b/writer/pom.xml @@ -20,10 +20,6 @@ io.github.dfa1.vortex vortex-core - - com.google.flatbuffers - flatbuffers-java - io.airlift aircompressor-v3 diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index c6df743bb..d405d77be 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.writer; -import com.google.flatbuffers.FlatBufferBuilder; +import io.github.dfa1.vortex.fbsrt.FbsBuilder; import io.github.dfa1.vortex.writer.encode.DateTimePartsData; import io.github.dfa1.vortex.writer.encode.FixedSizeListData; import io.github.dfa1.vortex.writer.encode.ListData; @@ -11,7 +11,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.writer.encode.EncodeContext; import io.github.dfa1.vortex.writer.encode.EncodeNode; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.writer.encode.EncodeResult; import io.github.dfa1.vortex.writer.encode.NullableData; import io.github.dfa1.vortex.writer.encode.StructData; @@ -35,15 +35,15 @@ import io.github.dfa1.vortex.writer.encode.SparseEncodingEncoder; import io.github.dfa1.vortex.writer.encode.VarBinEncodingEncoder; import io.github.dfa1.vortex.writer.encode.ZstdEncodingEncoder; -import io.github.dfa1.vortex.fbs.ArraySpec; -import io.github.dfa1.vortex.fbs.Extension; -import io.github.dfa1.vortex.fbs.Footer; -import io.github.dfa1.vortex.fbs.Layout; -import io.github.dfa1.vortex.fbs.LayoutSpec; -import io.github.dfa1.vortex.fbs.Postscript; -import io.github.dfa1.vortex.fbs.PostscriptSegment; -import io.github.dfa1.vortex.fbs.SegmentSpec; -import io.github.dfa1.vortex.fbs.Type; +import io.github.dfa1.vortex.fbs.FbsArraySpec; +import io.github.dfa1.vortex.fbs.FbsExtension; +import io.github.dfa1.vortex.fbs.FbsFooter; +import io.github.dfa1.vortex.fbs.FbsLayout; +import io.github.dfa1.vortex.fbs.FbsLayoutSpec; +import io.github.dfa1.vortex.fbs.FbsPostscript; +import io.github.dfa1.vortex.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.fbs.FbsSegmentSpec; +import io.github.dfa1.vortex.fbs.FbsType; import java.io.Closeable; import java.io.IOException; @@ -74,7 +74,7 @@ /// ``` public final class VortexWriter implements Closeable { - // Indices into layout_specs list in the Footer + // Indices into layout_specs list in the FbsFooter private static final int LAYOUT_FLAT = 0; private static final int LAYOUT_CHUNKED = 1; private static final int LAYOUT_STRUCT = 2; @@ -120,7 +120,7 @@ public final class VortexWriter implements Closeable { // Per-column zone-maps, populated by flushZoneMaps() in close() when enableZoneMaps is set. private final Map zoneMaps = new LinkedHashMap<>(); - // Stats (ScalarValue bytes) of the most recently written segment, captured for ChunkRef. + // Stats (ProtoScalarValue bytes) of the most recently written segment, captured for ChunkRef. private byte[] lastStatsMin; private byte[] lastStatsMax; private byte[] lastStatsSum; @@ -157,7 +157,7 @@ private static WriteRegistry buildRegistry(List encoders) { private static List buildCascadeCodecs(WriteOptions options) { List codecs = new ArrayList<>(); - // Extension-dtype dispatch order matters: findPrimitiveEncoding picks the first + // FbsExtension-dtype dispatch order matters: findPrimitiveEncoding picks the first // accepting codec. DateTimePartsEncoding goes first because it consumes // pre-decomposed DateTimePartsData (Parquet importer path); when the data is // raw primitive storage (JDBC's long[] via TimestampExtension.encodeAll) it @@ -272,27 +272,27 @@ private static long arrayLength(Object data) { } private static ByteBuffer buildDType(DType dtype) { - var fbb = new FlatBufferBuilder(128); + var fbb = new FbsBuilder(128); int off = serializeDType(fbb, dtype); - io.github.dfa1.vortex.fbs.DType.finishDTypeBuffer(fbb, off); - return fbb.dataBuffer().slice(fbb.dataBuffer().position(), fbb.dataBuffer().remaining()); + io.github.dfa1.vortex.fbs.FbsDType.finishFbsDTypeBuffer(fbb, off); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } - private static int serializeDType(FlatBufferBuilder fbb, DType dtype) { + private static int serializeDType(FbsBuilder fbb, DType dtype) { return switch (dtype) { case DType.Null _ -> { - io.github.dfa1.vortex.fbs.Null.startNull(fbb); - int inner = io.github.dfa1.vortex.fbs.Null.endNull(fbb); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Null, inner); + io.github.dfa1.vortex.fbs.FbsNull.startFbsNull(fbb); + int inner = io.github.dfa1.vortex.fbs.FbsNull.endFbsNull(fbb); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsNull, inner); } case DType.Bool b -> { - int inner = io.github.dfa1.vortex.fbs.Bool.createBool(fbb, b.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Bool, inner); + int inner = io.github.dfa1.vortex.fbs.FbsBool.createFbsBool(fbb, b.nullable()); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsBool, inner); } case DType.Primitive p -> { - int inner = io.github.dfa1.vortex.fbs.Primitive.createPrimitive( + int inner = io.github.dfa1.vortex.fbs.FbsPrimitive.createFbsPrimitive( fbb, p.ptype().ordinal(), p.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Primitive, inner); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, inner); } case DType.Struct s -> { // Build child DType tables first (FlatBuffers bottom-up requirement) @@ -304,42 +304,41 @@ private static int serializeDType(FlatBufferBuilder fbb, DType dtype) { for (int i = 0; i < nameOffsets.length; i++) { nameOffsets[i] = fbb.createString(s.fieldNames().get(i)); } - int namesVec = io.github.dfa1.vortex.fbs.Struct_.createNamesVector(fbb, nameOffsets); - int dtypesVec = io.github.dfa1.vortex.fbs.Struct_.createDtypesVector(fbb, fieldOffsets); - int inner = io.github.dfa1.vortex.fbs.Struct_.createStruct_( + int namesVec = io.github.dfa1.vortex.fbs.FbsStruct_.createNamesVector(fbb, nameOffsets); + int dtypesVec = io.github.dfa1.vortex.fbs.FbsStruct_.createDtypesVector(fbb, fieldOffsets); + int inner = io.github.dfa1.vortex.fbs.FbsStruct_.createFbsStruct_( fbb, namesVec, dtypesVec, s.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Struct_, inner); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsStruct_, inner); } case DType.Utf8 u -> { - int inner = io.github.dfa1.vortex.fbs.Utf8.createUtf8(fbb, u.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Utf8, inner); + int inner = io.github.dfa1.vortex.fbs.FbsUtf8.createFbsUtf8(fbb, u.nullable()); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsUtf8, inner); } case DType.List l -> { int elemTypeOff = serializeDType(fbb, l.elementType()); - int inner = io.github.dfa1.vortex.fbs.List.createList(fbb, elemTypeOff, l.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.List, inner); + int inner = io.github.dfa1.vortex.fbs.FbsList.createFbsList(fbb, elemTypeOff, l.nullable()); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsList, inner); } case DType.FixedSizeList fsl -> { int elemTypeOff = serializeDType(fbb, fsl.elementType()); - int inner = io.github.dfa1.vortex.fbs.FixedSizeList.createFixedSizeList( + int inner = io.github.dfa1.vortex.fbs.FbsFixedSizeList.createFbsFixedSizeList( fbb, elemTypeOff, fsl.fixedSize(), fsl.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.FixedSizeList, inner); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsFixedSizeList, inner); } case DType.Extension e -> { int idOff = fbb.createString(e.extensionId()); int storageDtypeOff = serializeDType(fbb, e.storageDType()); int metaOff = 0; if (e.metadata() != null) { - byte[] metaBytes = new byte[e.metadata().remaining()]; - e.metadata().duplicate().get(metaBytes); - metaOff = Extension.createMetadataVector(fbb, metaBytes); + byte[] metaBytes = e.metadata().toArray(java.lang.foreign.ValueLayout.JAVA_BYTE); + metaOff = FbsExtension.createMetadataVector(fbb, metaBytes); } - int inner = Extension.createExtension(fbb, idOff, storageDtypeOff, metaOff); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Extension, inner); + int inner = FbsExtension.createFbsExtension(fbb, idOff, storageDtypeOff, metaOff); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsExtension, inner); } case DType.Variant v -> { - int inner = io.github.dfa1.vortex.fbs.Variant.createVariant(fbb, v.nullable()); - yield io.github.dfa1.vortex.fbs.DType.createDType(fbb, Type.Variant, inner); + int inner = io.github.dfa1.vortex.fbs.FbsVariant.createFbsVariant(fbb, v.nullable()); + yield io.github.dfa1.vortex.fbs.FbsDType.createFbsDType(fbb, FbsType.FbsVariant, inner); } default -> throw new UnsupportedOperationException("unsupported DType: " + dtype); }; @@ -350,18 +349,18 @@ private static ByteBuffer buildPostscript( long dtypeOff, int dtypeLen, long layoutOff, int layoutLen ) { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); - int footerSegOff = PostscriptSegment.createPostscriptSegment( + int footerSegOff = FbsPostscriptSegment.createFbsPostscriptSegment( fbb, footerOff, footerLen, 0, 0, 0); - int dtypeSegOff = PostscriptSegment.createPostscriptSegment( + int dtypeSegOff = FbsPostscriptSegment.createFbsPostscriptSegment( fbb, dtypeOff, dtypeLen, 0, 0, 0); - int layoutSegOff = PostscriptSegment.createPostscriptSegment( + int layoutSegOff = FbsPostscriptSegment.createFbsPostscriptSegment( fbb, layoutOff, layoutLen, 0, 0, 0); - int psOff = Postscript.createPostscript(fbb, dtypeSegOff, layoutSegOff, 0, footerSegOff); - Postscript.finishPostscriptBuffer(fbb, psOff); - return fbb.dataBuffer().slice(fbb.dataBuffer().position(), fbb.dataBuffer().remaining()); + int psOff = FbsPostscript.createFbsPostscript(fbb, dtypeSegOff, layoutSegOff, 0, footerSegOff); + FbsPostscript.finishFbsPostscriptBuffer(fbb, psOff); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } // ── Segment encoding ───────────────────────────────────────────────────── @@ -519,7 +518,7 @@ private int writeSegment(DType dtype, Object data) throws IOException { /// cannot unwrap). private int writeSegment(DType dtype, Object data, EncodingEncoder encodingOverride) throws IOException { // Non-extension nullable columns (Primitive, Utf8) wrap with MaskedEncodingEncoder here. - // Extension columns route through ExtEncodingEncoder.encode which itself delegates to + // FbsExtension columns route through ExtEncodingEncoder.encode which itself delegates to // MaskedEncodingEncoder when its storage data is NullableData — handled inside ExtEncoding. if (encodingOverride == null && data instanceof io.github.dfa1.vortex.writer.encode.NullableData @@ -621,35 +620,35 @@ private void writePadding(int n) throws IOException { } private ByteBuffer buildArrayFlatBuffer(EncodeResult result, long nullCount) { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); // Stats for the root node only (build vectors before the ArrayStats table). null_count is // always recorded; min/max only when the encoder produced them. int minVec = result.hasStats() - ? io.github.dfa1.vortex.fbs.ArrayStats.createMinVector(fbb, result.statsMin()) : 0; + ? io.github.dfa1.vortex.fbs.FbsArrayStats.createMinVector(fbb, result.statsMin()) : 0; int maxVec = result.hasStats() - ? io.github.dfa1.vortex.fbs.ArrayStats.createMaxVector(fbb, result.statsMax()) : 0; + ? io.github.dfa1.vortex.fbs.FbsArrayStats.createMaxVector(fbb, result.statsMax()) : 0; // forceDefaults only while building ArrayStats, so null_count = 0 is serialised (flatbuffers // omits a scalar equal to its default otherwise) — matching the Rust writer and letting the // reader prune IS NULL on zero-null chunks. Reset immediately so the Array/ArrayNode tables // keep their normal (offset-default-omitting) layout. fbb.forceDefaults(true); - io.github.dfa1.vortex.fbs.ArrayStats.startArrayStats(fbb); + io.github.dfa1.vortex.fbs.FbsArrayStats.startFbsArrayStats(fbb); if (result.hasStats()) { - io.github.dfa1.vortex.fbs.ArrayStats.addMin(fbb, minVec); - io.github.dfa1.vortex.fbs.ArrayStats.addMax(fbb, maxVec); + io.github.dfa1.vortex.fbs.FbsArrayStats.addMin(fbb, minVec); + io.github.dfa1.vortex.fbs.FbsArrayStats.addMax(fbb, maxVec); } - io.github.dfa1.vortex.fbs.ArrayStats.addNullCount(fbb, nullCount); - int statsOff = io.github.dfa1.vortex.fbs.ArrayStats.endArrayStats(fbb); + io.github.dfa1.vortex.fbs.FbsArrayStats.addNullCount(fbb, nullCount); + int statsOff = io.github.dfa1.vortex.fbs.FbsArrayStats.endFbsArrayStats(fbb); fbb.forceDefaults(false); int rootNodeOff = buildArrayNodeFlatBuffer(fbb, result.rootNode(), statsOff); // Buffer struct vector — one entry per buffer in result. - // Layout (LE): padding(u16) | alignment_exponent(u8) | compression(u8) | length(u32) + // FbsLayout (LE): padding(u16) | alignment_exponent(u8) | compression(u8) | length(u32) // FlatBuffers builds backward: iterate in reverse. var bufs = result.buffers(); - io.github.dfa1.vortex.fbs.Array.startBuffersVector(fbb, bufs.size()); + io.github.dfa1.vortex.fbs.FbsArray.startBuffersVector(fbb, bufs.size()); for (int i = bufs.size() - 1; i >= 0; i--) { fbb.prep(4, 8); fbb.putInt((int) bufs.get(i).byteSize()); @@ -659,14 +658,14 @@ private ByteBuffer buildArrayFlatBuffer(EncodeResult result, long nullCount) { } int bufVec = fbb.endVector(); - int arrayOff = io.github.dfa1.vortex.fbs.Array.createArray(fbb, rootNodeOff, bufVec); - io.github.dfa1.vortex.fbs.Array.finishArrayBuffer(fbb, arrayOff); - return fbb.dataBuffer().slice(fbb.dataBuffer().position(), fbb.dataBuffer().remaining()); + int arrayOff = io.github.dfa1.vortex.fbs.FbsArray.createFbsArray(fbb, rootNodeOff, bufVec); + io.github.dfa1.vortex.fbs.FbsArray.finishFbsArrayBuffer(fbb, arrayOff); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } - // ── Footer / metadata serialization ────────────────────────────────────── + // ── FbsFooter / metadata serialization ────────────────────────────────────── - private int buildArrayNodeFlatBuffer(FlatBufferBuilder fbb, EncodeNode node, int statsOff) { + private int buildArrayNodeFlatBuffer(FbsBuilder fbb, EncodeNode node, int statsOff) { // Build children first (FlatBuffers bottom-up: nested objects before parent table) int[] childOffsets = new int[node.children().length]; for (int i = 0; i < childOffsets.length; i++) { @@ -674,20 +673,19 @@ private int buildArrayNodeFlatBuffer(FlatBufferBuilder fbb, EncodeNode node, int } int metaOff = 0; - if (node.metadata() != null && node.metadata().hasRemaining()) { - byte[] metaBytes = new byte[node.metadata().remaining()]; - node.metadata().duplicate().get(metaBytes); - metaOff = io.github.dfa1.vortex.fbs.ArrayNode.createMetadataVector(fbb, metaBytes); + if (node.metadata() != null && node.metadata().byteSize() > 0) { + byte[] metaBytes = node.metadata().toArray(java.lang.foreign.ValueLayout.JAVA_BYTE); + metaOff = io.github.dfa1.vortex.fbs.FbsArrayNode.createMetadataVector(fbb, metaBytes); } int childVec = 0; if (childOffsets.length > 0) { - childVec = io.github.dfa1.vortex.fbs.ArrayNode.createChildrenVector(fbb, childOffsets); + childVec = io.github.dfa1.vortex.fbs.FbsArrayNode.createChildrenVector(fbb, childOffsets); } - int bufIdxVec = io.github.dfa1.vortex.fbs.ArrayNode.createBuffersVector(fbb, node.bufferIndices()); + int bufIdxVec = io.github.dfa1.vortex.fbs.FbsArrayNode.createBuffersVector(fbb, node.bufferIndices()); int encIdx = encodingIdx.get(node.encodingId()); - return io.github.dfa1.vortex.fbs.ArrayNode.createArrayNode( + return io.github.dfa1.vortex.fbs.FbsArrayNode.createFbsArrayNode( fbb, encIdx, metaOff, childVec, bufIdxVec, statsOff); } @@ -788,16 +786,16 @@ private void emitZoneMap(String colName, DType minMaxDtype, List minByte /// Wraps a column's data layout in a `vortex.stats` (zoned) layout when a zone-map was /// emitted for it; otherwise returns the data layout unchanged. - private int wrapZoneMap(FlatBufferBuilder fbb, String colName, int dataLayout, long colRows) { + private int wrapZoneMap(FbsBuilder fbb, String colName, int dataLayout, long colRows) { ZoneMapRef zm = zoneMaps.get(colName); if (zm == null) { return dataLayout; } - int zonesSegV = Layout.createSegmentsVector(fbb, new long[]{zm.zonesSegIdx()}); - int zonesFlat = Layout.createLayout(fbb, LAYOUT_FLAT, zm.nZones(), 0, 0, zonesSegV); - int childV = Layout.createChildrenVector(fbb, new int[]{dataLayout, zonesFlat}); - int metaV = Layout.createMetadataVector(fbb, zonedMetadataBytes(zm.zoneLen(), zm.hasMinMax(), zm.hasSum())); - return Layout.createLayout(fbb, LAYOUT_ZONED, colRows, metaV, childV, 0); + int zonesSegV = FbsLayout.createSegmentsVector(fbb, new long[]{zm.zonesSegIdx()}); + int zonesFlat = FbsLayout.createFbsLayout(fbb, LAYOUT_FLAT, zm.nZones(), 0, 0, zonesSegV); + int childV = FbsLayout.createChildrenVector(fbb, new int[]{dataLayout, zonesFlat}); + int metaV = FbsLayout.createMetadataVector(fbb, zonedMetadataBytes(zm.zoneLen(), zm.hasMinMax(), zm.hasSum())); + return FbsLayout.createFbsLayout(fbb, LAYOUT_ZONED, colRows, metaV, childV, 0); } /// `vortex.stats` metadata: `u32` zone length (LE) + a 1-byte stat bitset (LSB-first) with the @@ -868,7 +866,7 @@ private static byte[] columnSum(DType dtype, Object data) { } /// Builds the per-zone min (or max) values array for the resolved min/max `dtype`, decoding each - /// zone's serialised [ScalarValue] stat into the array shape its encoder expects. + /// zone's serialised [ProtoScalarValue] stat into the array shape its encoder expects. private static Object zoneStatValues(DType minMaxDtype, List statBytes) throws IOException { return switch (minMaxDtype) { case DType.Primitive p -> statColumn(p.ptype(), statBytes); @@ -899,7 +897,7 @@ private static Object sumColumn(DType sumDtype, List sumBytes, boolean[] return a; } - /// Builds the per-zone string array by decoding each zone's serialised string [ScalarValue] + /// Builds the per-zone string array by decoding each zone's serialised string [ProtoScalarValue] /// stat. Used for Utf8 columns whose `vortex.varbin` encoder records full string min/max scalars. private static String[] statStringColumn(List statBytes) throws IOException { String[] out = new String[statBytes.size()]; @@ -910,7 +908,7 @@ private static String[] statStringColumn(List statBytes) throws IOExcept } /// Builds the per-zone values array in the storage shape the primitive encoder expects, decoding - /// each zone's serialised [ScalarValue] stat. + /// each zone's serialised [ProtoScalarValue] stat. private static Object statColumn(PType ptype, List statBytes) throws IOException { int n = statBytes.size(); return switch (ptype) { @@ -969,27 +967,27 @@ private static Object statColumn(PType ptype, List statBytes) throws IOE private static long scalarLong(byte[] bytes) throws IOException { // Integer columns serialise min/max as int64 (signed) or uint64 (unsigned). - ScalarValue sv = decodeScalar(bytes); + ProtoScalarValue sv = decodeScalar(bytes); return sv.int64_value() != null ? sv.int64_value() : sv.uint64_value(); } private static double scalarDouble(byte[] bytes) throws IOException { // Float columns serialise min/max as f64 (F64) or f32 (F32). Branch rather than use a // ternary so the F32 path widens Float -> double explicitly instead of mixing boxed types. - ScalarValue sv = decodeScalar(bytes); + ProtoScalarValue sv = decodeScalar(bytes); if (sv.f64_value() != null) { return sv.f64_value(); } return sv.f32_value(); } - private static ScalarValue decodeScalar(byte[] bytes) throws IOException { + private static ProtoScalarValue decodeScalar(byte[] bytes) throws IOException { MemorySegment seg = MemorySegment.ofArray(bytes); - return ScalarValue.decode(seg, 0, seg.byteSize()); + return ProtoScalarValue.decode(seg, 0, seg.byteSize()); } private ByteBuffer buildFooter() { - var fbb = new FlatBufferBuilder(512); + var fbb = new FbsBuilder(512); // array_specs: all encoding IDs used across all written segments, in registration order EncodingId[] encIds = encodingIdx.entrySet().stream() @@ -998,33 +996,33 @@ private ByteBuffer buildFooter() { .toArray(EncodingId[]::new); int[] asOffsets = new int[encIds.length]; for (int i = 0; i < encIds.length; i++) { - asOffsets[i] = ArraySpec.createArraySpec(fbb, fbb.createString(encIds[i].id())); + asOffsets[i] = FbsArraySpec.createFbsArraySpec(fbb, fbb.createString(encIds[i].id())); } - int asv = Footer.createArraySpecsVector(fbb, asOffsets); + int asv = FbsFooter.createArraySpecsVector(fbb, asOffsets); // layout_specs: ["vortex.flat", "vortex.chunked", "vortex.struct", "vortex.dict"] - int ls0 = LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.flat")); - int ls1 = LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.chunked")); - int ls2 = LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.struct")); - int ls3 = LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.dict")); - int ls4 = LayoutSpec.createLayoutSpec(fbb, fbb.createString("vortex.stats")); - int lsv = Footer.createLayoutSpecsVector(fbb, new int[]{ls0, ls1, ls2, ls3, ls4}); + int ls0 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.flat")); + int ls1 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.chunked")); + int ls2 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.struct")); + int ls3 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.dict")); + int ls4 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.stats")); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, new int[]{ls0, ls1, ls2, ls3, ls4}); // segment_specs (inline struct vector — write in reverse order) - Footer.startSegmentSpecsVector(fbb, segs.size()); + FbsFooter.startSegmentSpecsVector(fbb, segs.size()); for (int i = segs.size() - 1; i >= 0; i--) { SegRef s = segs.get(i); - SegmentSpec.createSegmentSpec(fbb, s.offset(), s.len(), 6, 0, 0); + FbsSegmentSpec.createFbsSegmentSpec(fbb, s.offset(), s.len(), 6, 0, 0); } int ssv = fbb.endVector(); - int off = Footer.createFooter(fbb, asv, lsv, ssv, 0, 0); + int off = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); fbb.finish(off); - return fbb.dataBuffer().slice(fbb.dataBuffer().position(), fbb.dataBuffer().remaining()); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } private ByteBuffer buildLayout() { - var fbb = new FlatBufferBuilder(256); + var fbb = new FbsBuilder(256); int colCount = schema.fieldNames().size(); int[] colLayouts = new int[colCount]; @@ -1045,12 +1043,12 @@ private ByteBuffer buildLayout() { int[] flats = new int[chunks.size()]; for (int i = 0; i < chunks.size(); i++) { ChunkRef cr = chunks.get(i); - int segV = Layout.createSegmentsVector(fbb, new long[]{cr.segIdx()}); - flats[i] = Layout.createLayout(fbb, LAYOUT_FLAT, cr.rowCount(), 0, 0, segV); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{cr.segIdx()}); + flats[i] = FbsLayout.createFbsLayout(fbb, LAYOUT_FLAT, cr.rowCount(), 0, 0, segV); colRows += cr.rowCount(); } - int childV = Layout.createChildrenVector(fbb, flats); - int dataChunked = Layout.createLayout(fbb, LAYOUT_CHUNKED, colRows, 0, childV, 0); + int childV = FbsLayout.createChildrenVector(fbb, flats); + int dataChunked = FbsLayout.createFbsLayout(fbb, LAYOUT_CHUNKED, colRows, 0, childV, 0); colLayouts[c] = wrapZoneMap(fbb, colName, dataChunked, colRows); if (totalRows == 0) { totalRows = colRows; @@ -1058,39 +1056,39 @@ private ByteBuffer buildLayout() { } } - int rootChildV = Layout.createChildrenVector(fbb, colLayouts); - int rootLayout = Layout.createLayout(fbb, LAYOUT_STRUCT, totalRows, 0, rootChildV, 0); - Layout.finishLayoutBuffer(fbb, rootLayout); - return fbb.dataBuffer().slice(fbb.dataBuffer().position(), fbb.dataBuffer().remaining()); + int rootChildV = FbsLayout.createChildrenVector(fbb, colLayouts); + int rootLayout = FbsLayout.createFbsLayout(fbb, LAYOUT_STRUCT, totalRows, 0, rootChildV, 0); + FbsLayout.finishFbsLayoutBuffer(fbb, rootLayout); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } - private int buildDictColLayout(FlatBufferBuilder fbb, DictColRef ref) { + private int buildDictColLayout(FbsBuilder fbb, DictColRef ref) { // Build codes Chunked layout: children are one Flat per original chunk int numChunks = ref.codesSegIdxes().size(); int[] codesFlats = new int[numChunks]; long totalCodesRows = 0; for (int j = 0; j < numChunks; j++) { long rowCount = ref.chunkRowCounts().get(j); - int segV = Layout.createSegmentsVector(fbb, new long[]{ref.codesSegIdxes().get(j)}); - codesFlats[j] = Layout.createLayout(fbb, LAYOUT_FLAT, rowCount, 0, 0, segV); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.codesSegIdxes().get(j)}); + codesFlats[j] = FbsLayout.createFbsLayout(fbb, LAYOUT_FLAT, rowCount, 0, 0, segV); totalCodesRows += rowCount; } - int codesChildV = Layout.createChildrenVector(fbb, codesFlats); - int codesChunked = Layout.createLayout(fbb, LAYOUT_CHUNKED, totalCodesRows, 0, codesChildV, 0); + int codesChildV = FbsLayout.createChildrenVector(fbb, codesFlats); + int codesChunked = FbsLayout.createFbsLayout(fbb, LAYOUT_CHUNKED, totalCodesRows, 0, codesChildV, 0); // Build values Flat layout - int valSegV = Layout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()}); - int valuesFlat = Layout.createLayout(fbb, LAYOUT_FLAT, ref.valuesLen(), 0, 0, valSegV); + int valSegV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()}); + int valuesFlat = FbsLayout.createFbsLayout(fbb, LAYOUT_FLAT, ref.valuesLen(), 0, 0, valSegV); // DictLayoutMetadata proto (matches Rust): field 1 = codes_ptype (PType varint) PType codePType = codePTypeForSize((int) ref.valuesLen()); byte[] metaBytes = buildDictLayoutMetaBytes(codePType); - int metaVec = metaBytes.length > 0 ? Layout.createMetadataVector(fbb, metaBytes) : 0; + int metaVec = metaBytes.length > 0 ? FbsLayout.createMetadataVector(fbb, metaBytes) : 0; // Dict layout: child[0]=values, child[1]=codes (matches Rust DictLayout child order) int[] dictChildren = {valuesFlat, codesChunked}; - int dictChildV = Layout.createChildrenVector(fbb, dictChildren); - return Layout.createLayout(fbb, LAYOUT_DICT, totalCodesRows, metaVec, dictChildV, 0); + int dictChildV = FbsLayout.createChildrenVector(fbb, dictChildren); + return FbsLayout.createFbsLayout(fbb, LAYOUT_DICT, totalCodesRows, metaVec, dictChildV, 0); } private static byte[] buildDictLayoutMetaBytes(PType codePType) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java index 41b90dc52..ce9306e57 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java @@ -4,12 +4,11 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ALPMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoALPMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -203,9 +202,9 @@ private static EncodeResult encodeF64(double[] values, EncodeContext ctx) { EncodeNode encodedNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); if (d.patchIndices().isEmpty()) { - byte[] metaBytes = new ALPMetadata(d.expE(), d.expF(), null).encode(); + byte[] metaBytes = new ProtoALPMetadata(d.expE(), d.expF(), null).encode(); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), new EncodeNode[]{encodedNode}, new int[0]); + MemorySegment.ofArray(metaBytes), new EncodeNode[]{encodedNode}, new int[0]); return new EncodeResult(root, List.of(encodedBuf), d.statsMin(), d.statsMax()); } @@ -217,13 +216,13 @@ private static EncodeResult encodeF64(double[] values, EncodeContext ctx) { valBuf.setAtIndex(PTypeIO.LE_DOUBLE, i, d.patchValues().get(i)); } - PatchesMetadata patches = buildPatchesMeta(numPatches); - byte[] metaBytes = new ALPMetadata(d.expE(), d.expF(), patches).encode(); + ProtoPatchesMetadata patches = buildPatchesMeta(numPatches); + byte[] metaBytes = new ProtoALPMetadata(d.expE(), d.expF(), patches).encode(); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{encodedNode, idxNode, valNode}, new int[0]); return new EncodeResult(root, List.of(encodedBuf, idxBuf, valBuf), d.statsMin(), d.statsMax()); @@ -232,9 +231,9 @@ private static EncodeResult encodeF64(double[] values, EncodeContext ctx) { private static CascadeStep encodeCascadeF64(double[] values, EncodeContext ctx) { AlpF64Data d = computeF64(values); if (d.patchIndices().isEmpty()) { - byte[] metaBytes = new ALPMetadata(d.expE(), d.expF(), null).encode(); + byte[] metaBytes = new ProtoALPMetadata(d.expE(), d.expF(), null).encode(); EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), new EncodeNode[1], new int[0]); + MemorySegment.ofArray(metaBytes), new EncodeNode[1], new int[0]); ChildSlot slot = new ChildSlot(DType.I64, d.encodedArr(), 0); return new CascadeStep(partialRoot, List.of(), List.of(slot), d.statsMin(), d.statsMax(), true); } @@ -247,13 +246,13 @@ private static CascadeStep encodeCascadeF64(double[] values, EncodeContext ctx) valBuf.setAtIndex(PTypeIO.LE_DOUBLE, i, d.patchValues().get(i)); } - PatchesMetadata patches = buildPatchesMeta(numPatches); - byte[] metaBytes = new ALPMetadata(d.expE(), d.expF(), patches).encode(); + ProtoPatchesMetadata patches = buildPatchesMeta(numPatches); + byte[] metaBytes = new ProtoALPMetadata(d.expE(), d.expF(), patches).encode(); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), new EncodeNode[]{null, idxNode, valNode}, new int[0]); + MemorySegment.ofArray(metaBytes), new EncodeNode[]{null, idxNode, valNode}, new int[0]); ChildSlot slot = new ChildSlot(DType.I64, d.encodedArr(), 0); return new CascadeStep(partialRoot, List.of(idxBuf, valBuf), List.of(slot), d.statsMin(), d.statsMax(), true); } @@ -372,9 +371,9 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { EncodeNode encodedNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); if (patchIndices.isEmpty()) { - byte[] metaBytes = new ALPMetadata(expE, expF, null).encode(); + byte[] metaBytes = new ProtoALPMetadata(expE, expF, null).encode(); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), new EncodeNode[]{encodedNode}, new int[0]); + MemorySegment.ofArray(metaBytes), new EncodeNode[]{encodedNode}, new int[0]); return new EncodeResult(root, List.of(encodedBuf), statsMin, statsMax); } @@ -386,36 +385,36 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { valBuf.setAtIndex(PTypeIO.LE_FLOAT, i, patchValues.get(i)); } - PatchesMetadata patches = new PatchesMetadata( + ProtoPatchesMetadata patches = new ProtoPatchesMetadata( numPatches, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(PType.U32.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U32.ordinal()), null, null, null); - byte[] metaBytes = new ALPMetadata(expE, expF, patches).encode(); + byte[] metaBytes = new ProtoALPMetadata(expE, expF, patches).encode(); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{encodedNode, idxNode, valNode}, new int[0]); return new EncodeResult(root, List.of(encodedBuf, idxBuf, valBuf), statsMin, statsMax); } - private static PatchesMetadata buildPatchesMeta(int numPatches) { - return new PatchesMetadata( + private static ProtoPatchesMetadata buildPatchesMeta(int numPatches) { + return new ProtoPatchesMetadata( numPatches, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(PType.U32.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U32.ordinal()), null, null, null); } private static byte[] scalarF64(double v) { - return ScalarValue.ofF64Value(v).encode(); + return ProtoScalarValue.ofF64Value(v).encode(); } private static byte[] scalarF32(float v) { - return ScalarValue.ofF32Value(v).encode(); + return ProtoScalarValue.ofF32Value(v).encode(); } @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java index 491457211..24d4ce216 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java @@ -3,11 +3,10 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ALPRDMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPRDMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -246,7 +245,7 @@ private static EncodeResult buildEncodeResult( } EncodeNode[] children; - PatchesMetadata patchesMeta = null; + ProtoPatchesMetadata patchesMeta = null; if (excPos.isEmpty()) { children = new EncodeNode[]{leftNode, rightNode}; } else { @@ -267,23 +266,23 @@ private static EncodeResult buildEncodeResult( EncodeNode idxNode = EncodeNode.remapBufferIndices(idxResult.rootNode(), idxOffset); EncodeNode valNode = EncodeNode.remapBufferIndices(valResult.rootNode(), idxOffset + idxBufCount); - patchesMeta = new PatchesMetadata( + patchesMeta = new ProtoPatchesMetadata( excPos.size(), 0L, - io.github.dfa1.vortex.proto.PType.fromValue(PType.U64.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U64.ordinal()), null, null, null); children = new EncodeNode[]{leftNode, rightNode, idxNode, valNode}; } - byte[] metaBytes = new ALPRDMetadata( + byte[] metaBytes = new ProtoALPRDMetadata( rightBitWidth, dict.length, dictList, - io.github.dfa1.vortex.proto.PType.fromValue(PType.U16.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U16.ordinal()), patchesMeta ).encode(); EncodeNode root = new EncodeNode( - EncodingId.VORTEX_ALPRD, ByteBuffer.wrap(metaBytes), children, new int[]{}); + EncodingId.VORTEX_ALPRD, MemorySegment.ofArray(metaBytes), children, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); } @@ -300,15 +299,15 @@ private static EncodeResult emptyResult(DType rightDtype, EncodeContext ctx) { EncodeNode leftNode = EncodeNode.remapBufferIndices(leftResult.rootNode(), 0); EncodeNode rightNode = EncodeNode.remapBufferIndices(rightResult.rootNode(), leftBufCount); - byte[] metaBytes = new ALPRDMetadata( + byte[] metaBytes = new ProtoALPRDMetadata( 48, 0, List.of(), - io.github.dfa1.vortex.proto.PType.fromValue(PType.U16.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U16.ordinal()), null).encode(); EncodeNode root = new EncodeNode( - EncodingId.VORTEX_ALPRD, ByteBuffer.wrap(metaBytes), + EncodingId.VORTEX_ALPRD, MemorySegment.ofArray(metaBytes), new EncodeNode[]{leftNode, rightNode}, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java index 0cdb12beb..ff88df93c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java @@ -7,14 +7,13 @@ import io.github.dfa1.vortex.encoding.FastLanes; import io.github.dfa1.vortex.encoding.PrimitiveArrays; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.BitPackedMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -81,11 +80,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { if (hasNegative) { int wideBitWidth = typeBits; MemorySegment widePacked = packFastLanes(longs, n, wideBitWidth, typeBits, ctx.arena()); - byte[] wideMeta = new BitPackedMetadata(wideBitWidth, 0, null).encode(); + byte[] wideMeta = new ProtoBitPackedMetadata(wideBitWidth, 0, null).encode(); byte[] wideMin = statsBytes(ptype, signedMin); byte[] wideMax = statsBytes(ptype, signedMax); EncodeNode wideRoot = new EncodeNode(EncodingId.FASTLANES_BITPACKED, - ByteBuffer.wrap(wideMeta), new EncodeNode[0], new int[]{0}); + MemorySegment.ofArray(wideMeta), new EncodeNode[0], new int[]{0}); return new EncodeResult(wideRoot, List.of(widePacked), wideMin, wideMax); } @@ -98,8 +97,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { if (bitWidth >= typeBits) { // Packed width covers full type range — no overflow possible, no patches needed. - byte[] metaBytes = new BitPackedMetadata(bitWidth, 0, null).encode(); - EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, ByteBuffer.wrap(metaBytes), + byte[] metaBytes = new ProtoBitPackedMetadata(bitWidth, 0, null).encode(); + EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, MemorySegment.ofArray(metaBytes), new EncodeNode[0], new int[]{0}); return new EncodeResult(root, List.of(packed), statsMin, statsMax); } @@ -116,8 +115,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } if (patchIdx.isEmpty()) { - byte[] metaBytes = new BitPackedMetadata(bitWidth, 0, null).encode(); - EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, ByteBuffer.wrap(metaBytes), + byte[] metaBytes = new ProtoBitPackedMetadata(bitWidth, 0, null).encode(); + EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, MemorySegment.ofArray(metaBytes), new EncodeNode[0], new int[]{0}); return new EncodeResult(root, List.of(packed), statsMin, statsMax); } @@ -127,15 +126,15 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment idxBuf = buildPatchIdxBuf(patchIdx, idxPtype, ctx.arena()); MemorySegment valBuf = buildPatchValBuf(patchVal, ptype, ctx.arena()); - PatchesMetadata patches = new PatchesMetadata( + ProtoPatchesMetadata patches = new ProtoPatchesMetadata( numPatches, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(idxPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(idxPtype.ordinal()), null, null, null); - byte[] metaBytes = new BitPackedMetadata(bitWidth, 0, patches).encode(); + byte[] metaBytes = new ProtoBitPackedMetadata(bitWidth, 0, patches).encode(); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); - EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.FASTLANES_BITPACKED, MemorySegment.ofArray(metaBytes), new EncodeNode[]{idxNode, valNode}, new int[]{0}); return new EncodeResult(root, List.of(packed, idxBuf, valBuf), statsMin, statsMax); } @@ -243,9 +242,9 @@ private static MemorySegment packFastLanes(long[] values, int n, int bitWidth, i private static byte[] statsBytes(PType ptype, long value) { if (ptype.isUnsigned()) { - return ScalarValue.ofUint64Value(value).encode(); + return ProtoScalarValue.ofUint64Value(value).encode(); } - return ScalarValue.ofInt64Value(value).encode(); + return ProtoScalarValue.ofInt64Value(value).encode(); } private static long readWordFromSeg(MemorySegment seg, int off, int typeBits) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java index 2f16e1653..de510b2a8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -59,10 +59,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } } byte[] statsMin = bools.length > 0 - ? ScalarValue.ofBoolValue(!hasFalse).encode() + ? ProtoScalarValue.ofBoolValue(!hasFalse).encode() : null; byte[] statsMax = bools.length > 0 - ? ScalarValue.ofBoolValue(hasTrue).encode() + ? ProtoScalarValue.ofBoolValue(hasTrue).encode() : null; return EncodeResult.simple(encodingId(), encodeBool(bools, ctx.arena()), statsMin, statsMax); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoder.java index a5a15bd81..0763ab838 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoder.java @@ -12,7 +12,6 @@ import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -75,7 +74,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode root = new EncodeNode( EncodingId.VORTEX_CHUNKED, - ByteBuffer.wrap(new byte[0]), + MemorySegment.ofArray(new byte[0]), children, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 02e535ced..1e2f8ce80 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; @@ -54,7 +54,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array"); } long firstRaw = readFirstRaw(data, ptype); - ScalarValue scalar = buildScalar(ptype, firstRaw); + ProtoScalarValue scalar = buildScalar(ptype, firstRaw); return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, MemorySegment.ofArray(scalar.encode())); } @@ -106,12 +106,12 @@ private static boolean isConstant(Object data, PType ptype) { return true; } - private static ScalarValue buildScalar(PType ptype, long rawBits) { + private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { return switch (ptype) { - case U8, U16, U32, U64 -> ScalarValue.ofUint64Value(rawBits); - case I8, I16, I32, I64 -> ScalarValue.ofInt64Value(rawBits); - case F32 -> ScalarValue.ofF32Value(Float.intBitsToFloat((int) rawBits)); - case F64 -> ScalarValue.ofF64Value(Double.longBitsToDouble(rawBits)); + case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(rawBits); + case I8, I16, I32, I64 -> ProtoScalarValue.ofInt64Value(rawBits); + case F32 -> ProtoScalarValue.ofF32Value(Float.intBitsToFloat((int) rawBits)); + case F64 -> ProtoScalarValue.ofF64Value(Double.longBitsToDouble(rawBits)); default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); }; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java index 988264539..09b0c202d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.extension.ExtensionId; import io.github.dfa1.vortex.writer.ExtensionEncoder; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.time.LocalDate; import java.util.Collection; @@ -31,8 +31,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { // Rust vortex.date metadata: 1 byte = TimeUnit tag (Days = 4), required by Rust reader. - ByteBuffer meta = ByteBuffer.allocate(1); - meta.put(0, (byte) TimeUnit.Days.ordinal()); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) TimeUnit.Days.ordinal()}); return new DType.Extension( ExtensionId.VORTEX_DATE.id(), new DType.Primitive(PType.I32, nullable), diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index c720310c3..5e65a5f89 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -5,10 +5,9 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.TimeUnit; -import io.github.dfa1.vortex.proto.DateTimePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDateTimePartsMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -16,8 +15,8 @@ public final class DateTimePartsEncodingEncoder implements EncodingEncoder { private static final long SECONDS_PER_DAY = 86_400L; - private static final io.github.dfa1.vortex.proto.PType I64_PROTO = - io.github.dfa1.vortex.proto.PType.fromValue(PType.I64.ordinal()); + private static final io.github.dfa1.vortex.proto.ProtoPType I64_PROTO = + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I64.ordinal()); /// Public no-arg constructor required by [java.util.ServiceLoader]. public DateTimePartsEncodingEncoder() { @@ -38,13 +37,12 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { DType.Extension ext = (DType.Extension) dtype; DateTimePartsData d = (DateTimePartsData) data; - ByteBuffer extMeta = ext.metadata(); - if (extMeta == null || extMeta.remaining() < 3) { + MemorySegment extMeta = ext.metadata(); + if (extMeta == null || extMeta.byteSize() < 3) { throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "extension metadata missing or too short"); } - byte[] extBytes = new byte[extMeta.remaining()]; - extMeta.duplicate().get(extBytes); + byte[] extBytes = extMeta.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE); TimeUnit unit = TimeUnit.fromTag(extBytes[0]); long divisor = unit.divisor(); @@ -87,11 +85,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode secondsNode = EncodeNode.remapBufferIndices(secondsResult.rootNode(), off1); EncodeNode subsecondsNode = EncodeNode.remapBufferIndices(subsecondsResult.rootNode(), off2); - byte[] metaBytes = new DateTimePartsMetadata(I64_PROTO, I64_PROTO, I64_PROTO).encode(); + byte[] metaBytes = new ProtoDateTimePartsMetadata(I64_PROTO, I64_PROTO, I64_PROTO).encode(); EncodeNode root = new EncodeNode( EncodingId.VORTEX_DATETIMEPARTS, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{daysNode, secondsNode, subsecondsNode}, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); @@ -103,9 +101,8 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC return CascadeStep.notApplicable(); } DType.Extension ext = (DType.Extension) dtype; - ByteBuffer extMeta = ext.metadata(); - byte[] extBytes = new byte[extMeta.remaining()]; - extMeta.duplicate().get(extBytes); + MemorySegment extMeta = ext.metadata(); + byte[] extBytes = extMeta.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE); TimeUnit unit = TimeUnit.fromTag(extBytes[0]); long divisor = unit.divisor(); @@ -129,11 +126,11 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC subseconds[i] = rem % divisor; } - byte[] metaBytes = new DateTimePartsMetadata(I64_PROTO, I64_PROTO, I64_PROTO).encode(); + byte[] metaBytes = new ProtoDateTimePartsMetadata(I64_PROTO, I64_PROTO, I64_PROTO).encode(); EncodeNode partialRoot = new EncodeNode( EncodingId.VORTEX_DATETIMEPARTS, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[3], new int[0]); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoder.java index 7a8442826..8fa409020 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoder.java @@ -3,9 +3,9 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.DecimalBytePartsMetadata; +import java.lang.foreign.MemorySegment; +import io.github.dfa1.vortex.proto.ProtoDecimalBytePartsMetadata; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `vortex.decimal_byte_parts`. @@ -32,10 +32,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { DType mspDtype = new DType.Primitive(PType.I64, d.nullable()); EncodeResult mspResult = ctx.lookupEncoder(EncodingId.VORTEX_PRIMITIVE).encode(mspDtype, longs, ctx); - DecimalBytePartsMetadata proto = new DecimalBytePartsMetadata( - io.github.dfa1.vortex.proto.PType.fromValue(PType.I64.ordinal()), + ProtoDecimalBytePartsMetadata proto = new ProtoDecimalBytePartsMetadata( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I64.ordinal()), 0); - ByteBuffer metaBuf = ByteBuffer.wrap(proto.encode()); + MemorySegment metaBuf = MemorySegment.ofArray(proto.encode()); EncodeNode mspNode = EncodeNode.remapBufferIndices(mspResult.rootNode(), 0); EncodeNode root = new EncodeNode( diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoder.java index 067e4b912..3cc772d66 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoder.java @@ -3,10 +3,9 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.DecimalMetadata; +import io.github.dfa1.vortex.proto.ProtoDecimalMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `vortex.decimal`. @@ -36,7 +35,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { throw new VortexException(EncodingId.VORTEX_DECIMAL, "buffer size %d not multiple of byteWidth %d".formatted(seg.byteSize(), bw)); } - ByteBuffer metaBuf = ByteBuffer.wrap(new DecimalMetadata(valuesType).encode()); + MemorySegment metaBuf = MemorySegment.ofArray(new ProtoDecimalMetadata(valuesType).encode()); EncodeNode node = new EncodeNode(EncodingId.VORTEX_DECIMAL, metaBuf, new EncodeNode[0], new int[]{0}); return new EncodeResult(node, List.of(seg), null, null); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java index 3ea88778a..207d49ef4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java @@ -5,11 +5,10 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.FastLanes; import io.github.dfa1.vortex.encoding.PrimitiveArrays; -import io.github.dfa1.vortex.proto.DeltaMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoDeltaMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `fastlanes.delta`. @@ -94,14 +93,14 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment basesSeg = PrimitiveArrays.fromLongs(basesAll, ptype, ctx.arena()); MemorySegment deltasSeg = PrimitiveArrays.fromLongs(deltasAll, ptype, ctx.arena()); - byte[] metaBytes = new DeltaMetadata(paddedLen, 0).encode(); + byte[] metaBytes = new ProtoDeltaMetadata(paddedLen, 0).encode(); byte[] statsMin = n > 0 ? statsBytes(ptype, minVal) : null; byte[] statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; EncodeNode basesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode deltasNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); - EncodeNode root = new EncodeNode(EncodingId.FASTLANES_DELTA, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.FASTLANES_DELTA, MemorySegment.ofArray(metaBytes), new EncodeNode[]{basesNode, deltasNode}, new int[0]); return new EncodeResult(root, List.of(basesSeg, deltasSeg), statsMin, statsMax); } @@ -120,9 +119,9 @@ private static void deltaChunk(long[] transposed, long[] bases, int lanes, int t private static byte[] statsBytes(PType ptype, long value) { if (ptype.isUnsigned()) { - return ScalarValue.ofUint64Value(value).encode(); + return ProtoScalarValue.ofUint64Value(value).encode(); } - return ScalarValue.ofInt64Value(value).encode(); + return ProtoScalarValue.ofInt64Value(value).encode(); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java index 775ca7763..d482ffa28 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java @@ -5,14 +5,13 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.DictMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoDictMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; @@ -74,7 +73,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { writeCodeToSeg(codesBuf, codePType, i, readCodeFromArr(d.codesArr(), codePType, i)); } - ByteBuffer meta = ByteBuffer.allocate(1).put(0, (byte) codePType.ordinal()); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) codePType.ordinal()}); EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode codesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode rootNode = new EncodeNode( @@ -93,7 +92,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { DictData d = buildDictData(dtype, data, ctx); PType codePType = d.codePType(); - ByteBuffer meta = ByteBuffer.allocate(1).put(0, (byte) codePType.ordinal()); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) codePType.ordinal()}); EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode partialRoot = new EncodeNode( EncodingId.VORTEX_DICT, meta, @@ -143,32 +142,32 @@ private static EncodeResult encodeUtf8(String[] strings, EncodeContext ctx) { writeCodeToSeg(codesBuf, codePType, i, valueMap.get(strings[i])); } - byte[] metaBytes = new DictMetadata( + byte[] metaBytes = new ProtoDictMetadata( dictSize, - io.github.dfa1.vortex.proto.PType.fromValue(codePType.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(codePType.ordinal()), null, null ).encode(); - byte[] varBinMetaBytes = new VarBinMetadata( - io.github.dfa1.vortex.proto.PType.fromValue(PType.I64.ordinal()) + byte[] varBinMetaBytes = new ProtoVarBinMetadata( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I64.ordinal()) ).encode(); EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode valuesNode = new EncodeNode(EncodingId.VORTEX_VARBIN, - ByteBuffer.wrap(varBinMetaBytes), + MemorySegment.ofArray(varBinMetaBytes), new EncodeNode[]{offsetsNode}, new int[]{0}); EncodeNode codesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode( - EncodingId.VORTEX_DICT, ByteBuffer.wrap(metaBytes), + EncodingId.VORTEX_DICT, MemorySegment.ofArray(metaBytes), new EncodeNode[]{codesNode, valuesNode}, new int[0]); String minStr = valueMap.keySet().stream().min(String::compareTo).orElse(null); String maxStr = valueMap.keySet().stream().max(String::compareTo).orElse(null); - byte[] statsMin = minStr != null ? ScalarValue.ofStringValue(minStr).encode() : null; - byte[] statsMax = maxStr != null ? ScalarValue.ofStringValue(maxStr).encode() : null; + byte[] statsMin = minStr != null ? ProtoScalarValue.ofStringValue(minStr).encode() : null; + byte[] statsMax = maxStr != null ? ProtoScalarValue.ofStringValue(maxStr).encode() : null; return new EncodeResult(root, List.of(dictBytesBuf, dictOffsetsBuf, codesBuf), statsMin, statsMax); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java index a3fd560f8..6a508f4f1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; /// Describes the ArrayNode tree written into a flat segment's FlatBuffer. /// Mirrors ArrayNode for the encode path. @@ -14,7 +14,7 @@ @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record EncodeNode( EncodingId encodingId, - ByteBuffer metadata, + MemorySegment metadata, EncodeNode[] children, int[] bufferIndices ) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoder.java index 3459f54f6..5628ac31f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoder.java @@ -10,7 +10,6 @@ import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -49,7 +48,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode root = new EncodeNode( EncodingId.VORTEX_FIXED_SIZE_LIST, - ByteBuffer.wrap(new byte[0]), + MemorySegment.ofArray(new byte[0]), new EncodeNode[]{elemNode}, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java index a1e895575..f2e918c88 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java @@ -6,10 +6,9 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PrimitiveArrays; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `fastlanes.for` (Frame of Reference). @@ -40,7 +39,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { long ref = computeRef(longs, n); MemorySegment residuals = toResidualBuffer(longs, ref, ptype, ctx); - ByteBuffer meta = buildForMeta(ref, ptype); + MemorySegment meta = buildForMeta(ref, ptype); EncodeNode child = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode root = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[]{child}, new int[0]); @@ -63,7 +62,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC if (ref == 0L && ptype.isUnsigned()) { return CascadeStep.notApplicable(); } - ByteBuffer meta = buildForMeta(ref, ptype); + MemorySegment meta = buildForMeta(ref, ptype); EncodeNode partialRoot = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[1], new int[0]); ChildSlot slot = new ChildSlot(dtype, residualsAsNativeArray(longs, ref, ptype), 0); @@ -81,13 +80,13 @@ private static long computeRef(long[] longs, int n) { return ref; } - private static ByteBuffer buildForMeta(long ref, PType ptype) { + private static MemorySegment buildForMeta(long ref, PType ptype) { boolean unsigned = switch (ptype) { case U8, U16, U32, U64 -> true; default -> false; }; - ScalarValue scalar = unsigned ? ScalarValue.ofUint64Value(ref) : ScalarValue.ofInt64Value(ref); - return ByteBuffer.wrap(scalar.encode()); + ProtoScalarValue scalar = unsigned ? ProtoScalarValue.ofUint64Value(ref) : ProtoScalarValue.ofInt64Value(ref); + return MemorySegment.ofArray(scalar.encode()); } private static Object residualsAsNativeArray(long[] longs, long ref, PType ptype) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index 1924a200d..52a85e090 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -4,12 +4,11 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.FSSTMetadata; +import io.github.dfa1.vortex.proto.ProtoFSSTMetadata; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -113,16 +112,16 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { codesOffBuf.setAtIndex(PTypeIO.LE_INT, (long) i + 1, (int) off); } - byte[] metaBytes = new FSSTMetadata( - io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal()), - io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal()) + byte[] metaBytes = new ProtoFSSTMetadata( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal()) ).encode(); EncodeNode uncompLensNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 3); EncodeNode codesOffNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 4); EncodeNode root = new EncodeNode( EncodingId.VORTEX_FSST, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{uncompLensNode, codesOffNode}, new int[]{0, 1, 2}); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java index ff78afe01..880f8554b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java @@ -10,10 +10,9 @@ import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ListMetadata; +import io.github.dfa1.vortex.proto.ProtoListMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -59,14 +58,14 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount); long elementsLen = ld.offsets()[(int) ld.outerLen()]; - byte[] metaBytes = new ListMetadata( + byte[] metaBytes = new ProtoListMetadata( elementsLen, - io.github.dfa1.vortex.proto.PType.fromValue(PType.I64.ordinal()) + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I64.ordinal()) ).encode(); EncodeNode root = new EncodeNode( EncodingId.VORTEX_LIST, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{elemNode, offsetsNode}, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java index 3d67fec24..23f679b7c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java @@ -10,10 +10,9 @@ import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ListViewMetadata; +import io.github.dfa1.vortex.proto.ProtoListViewMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -67,15 +66,15 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode sizesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount + 1); long elementsLen = java.lang.reflect.Array.getLength(lvd.elements()); - byte[] metaBytes = new ListViewMetadata( + byte[] metaBytes = new ProtoListViewMetadata( elementsLen, - io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal()), - io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal()) + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal()) ).encode(); EncodeNode root = new EncodeNode( EncodingId.VORTEX_LISTVIEW, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{elemNode, offsetsNode, sizesNode}, new int[]{}); return new EncodeResult(root, List.copyOf(allBuffers), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java index 225e46e40..099bf95c6 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java @@ -7,10 +7,9 @@ import io.github.dfa1.vortex.encoding.FastLanes; import io.github.dfa1.vortex.encoding.PrimitiveArrays; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.PatchedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchedMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `vortex.patched`. @@ -75,9 +74,9 @@ static CascadeStep encodeCascade(DType dtype, Object data) { return CascadeStep.notApplicable(); } - byte[] metaBytes = new PatchedMetadata(pd.nPatches, N_LANES, 0).encode(); + byte[] metaBytes = new ProtoPatchedMetadata(pd.nPatches, N_LANES, 0).encode(); EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_PATCHED, - ByteBuffer.wrap(metaBytes), new EncodeNode[]{null, null, null, null}, new int[]{}); + MemorySegment.ofArray(metaBytes), new EncodeNode[]{null, null, null, null}, new int[]{}); DType u32Dtype = DType.U32; DType u16Dtype = DType.U16; @@ -128,13 +127,13 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { PTypeIO.set(patchValBuf, (long) i * elemBytes, ptype, pd.patchValues[i]); } - byte[] metaBytes = new PatchedMetadata(pd.nPatches, N_LANES, 0).encode(); + byte[] metaBytes = new ProtoPatchedMetadata(pd.nPatches, N_LANES, 0).encode(); EncodeNode innerNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode laneNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 3); - EncodeNode root = new EncodeNode(EncodingId.VORTEX_PATCHED, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.VORTEX_PATCHED, MemorySegment.ofArray(metaBytes), new EncodeNode[]{innerNode, laneNode, idxNode, valNode}, new int[]{}); return new EncodeResult(root, List.of(innerBuf, laneOffsBuf, patchIdxBuf, patchValBuf), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java index ec2b88ef9..9a1f45691 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java @@ -4,13 +4,12 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.PcoChunkInfo; -import io.github.dfa1.vortex.proto.PcoMetadata; -import io.github.dfa1.vortex.proto.PcoPageInfo; +import io.github.dfa1.vortex.proto.ProtoPcoChunkInfo; +import io.github.dfa1.vortex.proto.ProtoPcoMetadata; +import io.github.dfa1.vortex.proto.ProtoPcoPageInfo; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -89,7 +88,7 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { List chunkMetas = new ArrayList<>(); List pages = new ArrayList<>(); - List chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); int chunkStart = 0; while (chunkStart < n) { @@ -98,14 +97,14 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { ChunkResult result = encodeChunk(chunkLatents, ptype, dtypeSize, ctx.arena()); chunkMetas.add(result.chunkMeta()); pages.add(result.page()); - chunks.add(new PcoChunkInfo(List.of(new PcoPageInfo(result.pageN())))); + chunks.add(new ProtoPcoChunkInfo(List.of(new ProtoPcoPageInfo(result.pageN())))); chunkStart = chunkEnd; } List buffers = new ArrayList<>(chunkMetas); buffers.addAll(pages); int[] allBufIdxs = IntStream.range(0, buffers.size()).toArray(); - ByteBuffer metaBuf = buildMetadata(chunks); + MemorySegment metaBuf = buildMetadata(chunks); EncodeNode node = new EncodeNode(EncodingId.VORTEX_PCO, metaBuf, new EncodeNode[0], allBufIdxs); return new EncodeResult(node, buffers, null, null); } @@ -576,16 +575,16 @@ private static PcoWeightQuantizer.Result quantize( private static EncodeResult encodeEmpty() { byte[] header = {PCO_FORMAT_MAJOR, PCO_FORMAT_MINOR}; - PcoMetadata meta = new PcoMetadata(header, List.of()); - ByteBuffer metaBuf = ByteBuffer.wrap(meta.encode()); + ProtoPcoMetadata meta = new ProtoPcoMetadata(header, List.of()); + MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); EncodeNode node = new EncodeNode(EncodingId.VORTEX_PCO, metaBuf, new EncodeNode[0], new int[0]); return new EncodeResult(node, List.of(), null, null); } - private static ByteBuffer buildMetadata(List chunks) { + private static MemorySegment buildMetadata(List chunks) { byte[] header = {PCO_FORMAT_MAJOR, PCO_FORMAT_MINOR}; - PcoMetadata meta = new PcoMetadata(header, chunks); - return ByteBuffer.wrap(meta.encode()); + ProtoPcoMetadata meta = new ProtoPcoMetadata(header, chunks); + return MemorySegment.ofArray(meta.encode()); } // ── latent conversion ───────────────────────────────────────────────── diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index febd87cb5..825cda7e4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -86,7 +86,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are }; } - /// Computes the serialised min/max [io.github.dfa1.vortex.proto.ScalarValue] pair for a raw + /// Computes the serialised min/max [io.github.dfa1.vortex.proto.ProtoScalarValue] pair for a raw /// primitive array, in the same signed/unsigned/float shape the per-segment stats use. Returns /// `null` for an empty array. Shared so the dictionary zone-map path computes per-chunk min/max /// identically to the flat path. @@ -290,7 +290,7 @@ public static byte[][] minMaxStats(PType ptype, Object data) { }; } - /// Computes the serialised SUM [io.github.dfa1.vortex.proto.ScalarValue] for a raw primitive + /// Computes the serialised SUM [io.github.dfa1.vortex.proto.ProtoScalarValue] for a raw primitive /// array, in the widened shape Rust uses for zone-map sums: signed ints → `i64`, unsigned ints /// → `u64`, floats → `f64`. Returns `null` on integer overflow (Rust drops the zone's sum) and /// for an empty array. Floats never overflow to `null` (they saturate to infinity). @@ -436,18 +436,18 @@ public static byte[] sumStat(PType ptype, Object data) { } private static byte[] scalarI64(long v) { - return ScalarValue.ofInt64Value(v).encode(); + return ProtoScalarValue.ofInt64Value(v).encode(); } private static byte[] scalarU64(long v) { - return ScalarValue.ofUint64Value(v).encode(); + return ProtoScalarValue.ofUint64Value(v).encode(); } private static byte[] scalarF32(float v) { - return ScalarValue.ofF32Value(v).encode(); + return ProtoScalarValue.ofF32Value(v).encode(); } private static byte[] scalarF64(double v) { - return ScalarValue.ofF64Value(v).encode(); + return ProtoScalarValue.ofF64Value(v).encode(); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java index a291dd960..a94781473 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java @@ -6,11 +6,10 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PrimitiveArrays; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.RLEMetadata; +import io.github.dfa1.vortex.proto.ProtoRLEMetadata; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `fastlanes.rle`. @@ -85,12 +84,12 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { PType indicesPtype = PType.U16; PType offsetsPtype = PType.U64; - byte[] metaBytes = new RLEMetadata( + byte[] metaBytes = new ProtoRLEMetadata( globalValuesCount, paddedLen, - io.github.dfa1.vortex.proto.PType.fromValue(indicesPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(indicesPtype.ordinal()), numChunks, - io.github.dfa1.vortex.proto.PType.fromValue(offsetsPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(offsetsPtype.ordinal()), 0L ).encode(); @@ -99,7 +98,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode( EncodingId.FASTLANES_RLE, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{valuesNode, indicesNode, offsetsNode}, new int[0]); return new EncodeResult(root, List.of(valuesSeg, indicesSeg, offsetsSeg), null, null); @@ -129,12 +128,12 @@ private static EncodeResult encodeEmpty(EncodeContext ctx) { MemorySegment empty = ctx.arena().allocate(0); PType indicesPtype = PType.U16; PType offsetsPtype = PType.U64; - byte[] metaBytes = new RLEMetadata( + byte[] metaBytes = new ProtoRLEMetadata( 0L, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(indicesPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(indicesPtype.ordinal()), 0L, - io.github.dfa1.vortex.proto.PType.fromValue(offsetsPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(offsetsPtype.ordinal()), 0L ).encode(); EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); @@ -142,7 +141,7 @@ private static EncodeResult encodeEmpty(EncodeContext ctx) { EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode( EncodingId.FASTLANES_RLE, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{valuesNode, indicesNode, offsetsNode}, new int[0]); return new EncodeResult(root, List.of(empty, empty, empty), null, null); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index bed083de4..9b71328d1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -5,10 +5,9 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.RunEndMetadata; +import io.github.dfa1.vortex.proto.ProtoRunEndMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -90,15 +89,15 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { PTypeIO.set(valuesBuf, (long) i * elemBytes, ptype, values.get(i)); } - byte[] metaBytes = new RunEndMetadata( - io.github.dfa1.vortex.proto.PType.fromValue(PType.U32.ordinal()), + byte[] metaBytes = new ProtoRunEndMetadata( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.U32.ordinal()), numRuns, 0L ).encode(); EncodeNode endsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); - EncodeNode root = new EncodeNode(EncodingId.VORTEX_RUNEND, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.VORTEX_RUNEND, MemorySegment.ofArray(metaBytes), new EncodeNode[]{endsNode, valuesNode}, new int[0]); return new EncodeResult(root, List.of(endsBuf, valuesBuf), null, null); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index 0c62752a4..d5e41a493 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -4,10 +4,10 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SequenceMetadata; +import java.lang.foreign.MemorySegment; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSequenceMetadata; -import java.nio.ByteBuffer; import java.util.List; /// Write-only encoder for `vortex.sequence` — arithmetic sequences as (base, multiplier). @@ -55,8 +55,8 @@ private static EncodeResult encodeInteger(PType pt, Object data) { } } } - ScalarValue baseScalar = buildIntScalar(pt, base); - ScalarValue mulScalar = buildIntScalar(pt, multiplier); + ProtoScalarValue baseScalar = buildIntScalar(pt, base); + ProtoScalarValue mulScalar = buildIntScalar(pt, multiplier); return buildResult(baseScalar, mulScalar); } @@ -68,7 +68,7 @@ private static EncodeResult encodeF32(float[] data) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } } - return buildResult(ScalarValue.ofF32Value(base), ScalarValue.ofF32Value(mul)); + return buildResult(ProtoScalarValue.ofF32Value(base), ProtoScalarValue.ofF32Value(mul)); } private static EncodeResult encodeF64(double[] data) { @@ -79,7 +79,7 @@ private static EncodeResult encodeF64(double[] data) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } } - return buildResult(ScalarValue.ofF64Value(base), ScalarValue.ofF64Value(mul)); + return buildResult(ProtoScalarValue.ofF64Value(base), ProtoScalarValue.ofF64Value(mul)); } private static EncodeResult encodeF16(short[] data) { @@ -94,21 +94,21 @@ private static EncodeResult encodeF16(short[] data) { } } return buildResult( - ScalarValue.ofF16Value(Short.toUnsignedLong(baseShort)), - ScalarValue.ofF16Value(Short.toUnsignedLong(mulShort))); + ProtoScalarValue.ofF16Value(Short.toUnsignedLong(baseShort)), + ProtoScalarValue.ofF16Value(Short.toUnsignedLong(mulShort))); } - private static EncodeResult buildResult(ScalarValue base, ScalarValue mul) { - SequenceMetadata meta = new SequenceMetadata(base, mul); - ByteBuffer metaBuf = ByteBuffer.wrap(meta.encode()); + private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul) { + ProtoSequenceMetadata meta = new ProtoSequenceMetadata(base, mul); + MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); EncodeNode node = new EncodeNode(EncodingId.VORTEX_SEQUENCE, metaBuf, new EncodeNode[0], new int[]{}); return new EncodeResult(node, List.of(), null, null); } - private static ScalarValue buildIntScalar(PType pt, long value) { + private static ProtoScalarValue buildIntScalar(PType pt, long value) { return switch (pt) { - case U8, U16, U32, U64 -> ScalarValue.ofUint64Value(value); - default -> ScalarValue.ofInt64Value(value); + case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(value); + default -> ProtoScalarValue.ofInt64Value(value); }; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java index 101cd715e..1241ff687 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java @@ -5,12 +5,11 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SparseMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSparseMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -87,7 +86,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { int numPatches = patchIdx.size(); // Owned buffers: fill scalar. idx and val moved to ChildSlot so cascade can bitpack. - ScalarValue fillScalar = zeroScalar(ptype); + ProtoScalarValue fillScalar = zeroScalar(ptype); byte[] fillBytes = fillScalar.encode(); MemorySegment fillBuf = ctx.arena().allocate(fillBytes.length); MemorySegment.copy(MemorySegment.ofArray(fillBytes), 0, fillBuf, 0, fillBytes.length); @@ -95,17 +94,17 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { Object idxArr = idxArr(patchIdx, idxPtype); Object valArr = valArr(patchBits, ptype); - PatchesMetadata patchesMeta = new PatchesMetadata( + ProtoPatchesMetadata patchesMeta = new ProtoPatchesMetadata( numPatches, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(idxPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(idxPtype.ordinal()), null, null, null ); - byte[] metaBytes = new SparseMetadata(patchesMeta).encode(); + byte[] metaBytes = new ProtoSparseMetadata(patchesMeta).encode(); EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_SPARSE, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new EncodeNode[]{null, null}, new int[]{0}); DType idxDtype = new DType.Primitive(idxPtype, false); ChildSlot idxSlot = new ChildSlot(idxDtype, idxArr, 0); @@ -211,7 +210,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { int numPatches = patchIdx.size(); PType idxPtype = chooseIdxPtype(n); - ScalarValue fillScalar = zeroScalar(ptype); + ProtoScalarValue fillScalar = zeroScalar(ptype); byte[] fillBytes = fillScalar.encode(); MemorySegment fillBuf = ctx.arena().allocate(fillBytes.length); MemorySegment.copy(MemorySegment.ofArray(fillBytes), 0, fillBuf, 0, fillBytes.length); @@ -219,19 +218,19 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment idxBuf = buildIdxBuf(patchIdx, idxPtype, numPatches, ctx); MemorySegment valBuf = buildValBuf(patchBits, ptype, numPatches, ctx); - PatchesMetadata patchesMeta = new PatchesMetadata( + ProtoPatchesMetadata patchesMeta = new ProtoPatchesMetadata( numPatches, 0L, - io.github.dfa1.vortex.proto.PType.fromValue(idxPtype.ordinal()), + io.github.dfa1.vortex.proto.ProtoPType.fromValue(idxPtype.ordinal()), null, null, null ); - byte[] metaBytes = new SparseMetadata(patchesMeta).encode(); + byte[] metaBytes = new ProtoSparseMetadata(patchesMeta).encode(); EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); - EncodeNode root = new EncodeNode(EncodingId.VORTEX_SPARSE, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.VORTEX_SPARSE, MemorySegment.ofArray(metaBytes), new EncodeNode[]{idxNode, valNode}, new int[]{0}); return new EncodeResult(root, List.of(fillBuf, idxBuf, valBuf), null, null); } @@ -273,12 +272,12 @@ private static PType chooseIdxPtype(int n) { } } - private static ScalarValue zeroScalar(PType ptype) { + private static ProtoScalarValue zeroScalar(PType ptype) { return switch (ptype) { - case I8, I16, I32, I64 -> ScalarValue.ofInt64Value(0L); - case U8, U16, U32, U64 -> ScalarValue.ofUint64Value(0L); - case F32 -> ScalarValue.ofF32Value(0.0f); - case F64 -> ScalarValue.ofF64Value(0.0); + case I8, I16, I32, I64 -> ProtoScalarValue.ofInt64Value(0L); + case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(0L); + case F32 -> ProtoScalarValue.ofF32Value(0.0f); + case F64 -> ProtoScalarValue.ofF64Value(0.0); default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype: " + ptype); }; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java index 14d8e41a9..2452b24a7 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.extension.ExtensionId; import io.github.dfa1.vortex.writer.ExtensionEncoder; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.util.Collection; import java.util.UUID; @@ -34,7 +34,7 @@ public DType.Extension dtype(boolean nullable) { return new DType.Extension( ExtensionId.VORTEX_UUID.id(), new DType.FixedSizeList(u8, 16, nullable), - ByteBuffer.allocate(0), + MemorySegment.ofArray(new byte[0]), nullable); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java index fe1c31ceb..9ed7ea689 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java @@ -4,12 +4,11 @@ import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.List; @@ -54,19 +53,19 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { offsetsBuf.setAtIndex(PTypeIO.LE_LONG, (long) i + 1, pos); } - byte[] metaBytes = new VarBinMetadata(io.github.dfa1.vortex.proto.PType.fromValue(PType.I64.ordinal())).encode(); + byte[] metaBytes = new ProtoVarBinMetadata(io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I64.ordinal())).encode(); byte[][] stats = minMaxStats(strings); byte[] statsMin = stats != null ? stats[0] : null; byte[] statsMax = stats != null ? stats[1] : null; EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); - EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARBIN, ByteBuffer.wrap(metaBytes), + EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARBIN, MemorySegment.ofArray(metaBytes), new EncodeNode[]{offsetsNode}, new int[]{0}); return new EncodeResult(root, List.of(bytesBuf, offsetsBuf), statsMin, statsMax); } - /// Computes the serialised min/max string [ScalarValue] pair for a string array, skipping + /// Computes the serialised min/max string [ProtoScalarValue] pair for a string array, skipping /// `null` entries (lexicographic by [String#compareTo]). Returns `null` when every entry is /// `null`. Shared so the dictionary zone-map path computes per-chunk string min/max identically /// to the flat path. @@ -90,6 +89,6 @@ public static byte[][] minMaxStats(String[] strings) { if (minStr == null) { return null; } - return new byte[][]{ScalarValue.ofStringValue(minStr).encode(), ScalarValue.ofStringValue(maxStr).encode()}; + return new byte[][]{ProtoScalarValue.ofStringValue(minStr).encode(), ProtoScalarValue.ofStringValue(maxStr).encode()}; } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantData.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantData.java index 585cb7aa1..5bc53a1c1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantData.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantData.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.DType; -import io.github.dfa1.vortex.proto.Scalar; +import io.github.dfa1.vortex.proto.ProtoScalar; import java.util.Collections; import java.util.List; @@ -9,7 +9,7 @@ /// Input data for encoding a `vortex.variant` column. /// /// `values` holds one inner typed scalar per row, each wrapped as a variant value -/// (mirroring Rust `Scalar::variant(inner)`). The encoder coalesces adjacent equal +/// (mirroring Rust `ProtoScalar::variant(inner)`). The encoder coalesces adjacent equal /// values into constant runs: an all-equal column becomes a single `vortex.constant` /// child, while a varying column becomes a `vortex.chunked` of per-run constants. /// @@ -23,7 +23,7 @@ /// @param values one inner scalar per row, in row order /// @param shreddedData optional typed column data for the shredded child, or `null` /// @param shreddedDtype dtype of `shreddedData`, or `null` -public record VariantData(List values, Object shreddedData, DType shreddedDtype) { +public record VariantData(List values, Object shreddedData, DType shreddedDtype) { /// Validates and defensively copies the per-row values; rejects empty input and a /// half-specified shredded column. @@ -40,7 +40,7 @@ public record VariantData(List values, Object shreddedData, DType shredd /// Creates unshredded input from per-row scalars. /// /// @param values one inner scalar per row, in row order - public VariantData(List values) { + public VariantData(List values) { this(values, null, null); } @@ -49,7 +49,7 @@ public VariantData(List values) { /// @param length number of rows; must be positive /// @param value the inner scalar repeated on every row /// @return variant input describing a constant column - public static VariantData constant(int length, Scalar value) { + public static VariantData constant(int length, ProtoScalar value) { if (length <= 0) { throw new IllegalArgumentException("length must be positive, got " + length); } @@ -65,7 +65,7 @@ public static VariantData constant(int length, Scalar value) { /// @param shreddedData typed column data (encoder-input shape for `shreddedDtype`) /// @param shreddedDtype dtype of `shreddedData` /// @return variant input carrying a shredded child - public static VariantData shredded(List values, Object shreddedData, DType shreddedDtype) { + public static VariantData shredded(List values, Object shreddedData, DType shreddedDtype) { if (shreddedData == null || shreddedDtype == null) { throw new IllegalArgumentException("shreddedData and shreddedDtype are required"); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoder.java index 705f6e340..72822a9ab 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoder.java @@ -3,19 +3,18 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.Scalar; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.VariantMetadata; +import io.github.dfa1.vortex.proto.ProtoScalar; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoVariantMetadata; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; /// Write-only encoder for `vortex.variant`. /// /// Emits the canonical variant container: a single `core_storage` child holding the -/// full value per row, no shredded child, and `VariantMetadata` with no `shredded_dtype`. +/// full value per row, no shredded child, and `ProtoVariantMetadata` with no `shredded_dtype`. /// The container itself owns no buffers. /// /// `core_storage` is built from the per-row scalars in [VariantData], coalescing adjacent @@ -52,8 +51,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { "encode requires VariantData, got " + (data == null ? "null" : data.getClass().getName())); } - List values = variantData.values(); - List runValues = new ArrayList<>(); + List values = variantData.values(); + List runValues = new ArrayList<>(); List runLengths = new ArrayList<>(); coalesceRuns(values, runValues, runLengths); @@ -63,7 +62,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { : chunkedConstants(runValues, runLengths, ctx, buffers); EncodeNode[] children; - io.github.dfa1.vortex.proto.DType shreddedProto = null; + io.github.dfa1.vortex.proto.ProtoDType shreddedProto = null; if (variantData.shreddedData() != null) { children = new EncodeNode[]{coreStorage, encodeShredded(variantData, ctx, buffers)}; shreddedProto = toProtoDtype(variantData.shreddedDtype()); @@ -71,7 +70,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { children = new EncodeNode[]{coreStorage}; } - ByteBuffer containerMeta = ByteBuffer.wrap(new VariantMetadata(shreddedProto).encode()); + MemorySegment containerMeta = MemorySegment.ofArray(new ProtoVariantMetadata(shreddedProto).encode()); EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARIANT, containerMeta, children, new int[0]); return new EncodeResult(root, List.copyOf(buffers), null, null); } @@ -101,28 +100,28 @@ private static EncodeNode encodeShredded(VariantData data, EncodeContext ctx, Li return child; } - /// Converts a shreddable scalar dtype to its protobuf form for `VariantMetadata`. - private static io.github.dfa1.vortex.proto.DType toProtoDtype(DType dtype) { + /// Converts a shreddable scalar dtype to its protobuf form for `ProtoVariantMetadata`. + private static io.github.dfa1.vortex.proto.ProtoDType toProtoDtype(DType dtype) { return switch (dtype) { - case DType.Primitive p -> io.github.dfa1.vortex.proto.DType.ofPrimitive( - new io.github.dfa1.vortex.proto.Primitive( - io.github.dfa1.vortex.proto.PType.fromValue(p.ptype().ordinal()), p.nullable())); - case DType.Bool b -> io.github.dfa1.vortex.proto.DType.ofBool( - new io.github.dfa1.vortex.proto.Bool(b.nullable())); - case DType.Utf8 u -> io.github.dfa1.vortex.proto.DType.ofUtf8( - new io.github.dfa1.vortex.proto.Utf8(u.nullable())); - case DType.Binary bin -> io.github.dfa1.vortex.proto.DType.ofBinary( - new io.github.dfa1.vortex.proto.Binary(bin.nullable())); + case DType.Primitive p -> io.github.dfa1.vortex.proto.ProtoDType.ofPrimitive( + new io.github.dfa1.vortex.proto.ProtoPrimitive( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(p.ptype().ordinal()), p.nullable())); + case DType.Bool b -> io.github.dfa1.vortex.proto.ProtoDType.ofBool( + new io.github.dfa1.vortex.proto.ProtoBool(b.nullable())); + case DType.Utf8 u -> io.github.dfa1.vortex.proto.ProtoDType.ofUtf8( + new io.github.dfa1.vortex.proto.ProtoUtf8(u.nullable())); + case DType.Binary bin -> io.github.dfa1.vortex.proto.ProtoDType.ofBinary( + new io.github.dfa1.vortex.proto.ProtoBinary(bin.nullable())); default -> throw new VortexException(EncodingId.VORTEX_VARIANT, "shredded dtype not supported: " + dtype); }; } /// Groups adjacent equal scalars into runs, appending each run's value and length. - private static void coalesceRuns(List values, List runValues, List runLengths) { - Scalar prev = null; + private static void coalesceRuns(List values, List runValues, List runLengths) { + ProtoScalar prev = null; long runLen = 0; - for (Scalar s : values) { + for (ProtoScalar s : values) { if (prev != null && prev.equals(s)) { runLen++; } else { @@ -140,8 +139,8 @@ private static void coalesceRuns(List values, List runValues, Li /// Builds a buffer-backed `vortex.constant` child for one variant scalar, appending /// its serialized scalar to `buffers`. - private static EncodeNode constantChild(Scalar value, List buffers) { - ScalarValue scalar = ScalarValue.ofVariantValue(value); + private static EncodeNode constantChild(ProtoScalar value, List buffers) { + ProtoScalarValue scalar = ProtoScalarValue.ofVariantValue(value); int bufIdx = buffers.size(); buffers.add(MemorySegment.ofArray(scalar.encode())); return EncodeNode.leaf(EncodingId.VORTEX_CONSTANT, bufIdx); @@ -149,7 +148,7 @@ private static EncodeNode constantChild(Scalar value, List buffer /// Builds a `vortex.chunked` node: child 0 is the cumulative `u64` run offsets, the /// rest are one constant child per run. Appends all buffers to `buffers`. - private static EncodeNode chunkedConstants(List runValues, List runLengths, + private static EncodeNode chunkedConstants(List runValues, List runLengths, EncodeContext ctx, List buffers) { int nruns = runValues.size(); long[] offsets = new long[nruns + 1]; @@ -165,6 +164,6 @@ private static EncodeNode chunkedConstants(List runValues, List ru for (int i = 0; i < nruns; i++) { children[i + 1] = constantChild(runValues.get(i), buffers); } - return new EncodeNode(EncodingId.VORTEX_CHUNKED, ByteBuffer.wrap(new byte[0]), children, new int[0]); + return new EncodeNode(EncodingId.VORTEX_CHUNKED, MemorySegment.ofArray(new byte[0]), children, new int[0]); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index f86e475d8..61c6fa73c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -7,13 +7,12 @@ import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; -import io.github.dfa1.vortex.proto.ZstdFrameMetadata; -import io.github.dfa1.vortex.proto.ZstdMetadata; +import io.github.dfa1.vortex.proto.ProtoZstdFrameMetadata; +import io.github.dfa1.vortex.proto.ProtoZstdMetadata; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -60,11 +59,11 @@ private static EncodeResult encodeVarBin(String[] strings) { private static EncodeResult buildResult(byte[] raw, long n) { byte[] compressed = compress(raw); - byte[] meta = new ZstdMetadata( + byte[] meta = new ProtoZstdMetadata( 0, - java.util.List.of(new ZstdFrameMetadata(raw.length, n)) + java.util.List.of(new ProtoZstdFrameMetadata(raw.length, n)) ).encode(); - EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZSTD, ByteBuffer.wrap(meta), + EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new EncodeNode[0], new int[]{0}); return new EncodeResult(root, List.of(MemorySegment.ofArray(compressed)), null, null); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java index 006867647..e722841b6 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java @@ -1,5 +1,9 @@ package io.github.dfa1.vortex.writer; +import static io.github.dfa1.vortex.encoding.PTypeIO.LE_INT; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.PType; import io.github.dfa1.vortex.reader.Layout; @@ -16,8 +20,6 @@ import java.io.IOException; import java.lang.foreign.Arena; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -61,9 +63,9 @@ void enableZoneMaps_wrapsColumnInZonedLayoutWithMetadata(@TempDir Path tmp) thro // Then the column is a vortex.stats layout: [data, zones], zone_len=4, MAX+MIN+NULL_COUNT assertThat(column.isZoned()).isTrue(); assertThat(column.children()).hasSize(2); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.getInt(meta.position())).isEqualTo(4); // zone_len - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x78); // bits 3(MAX)+4(MIN)+5(SUM)+6(NULL_COUNT) + MemorySegment meta = column.metadata(); + assertThat(meta.get(LE_INT, 0)).isEqualTo(4); // zone_len + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x78); // bits 3(MAX)+4(MIN)+5(SUM)+6(NULL_COUNT) } } @@ -223,8 +225,8 @@ void chunkWithoutStats_emitsNullCountOnlyZoneMap(@TempDir Path tmp) throws IOExc try (VortexReader reader = VortexReader.open(file)) { Layout column = reader.layout().children().get(0); assertThat(column.isZoned()).isTrue(); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x60); + MemorySegment meta = column.metadata(); + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x60); } } @@ -247,8 +249,8 @@ void utf8Column_emitsStringMinMaxZoneMap(@TempDir Path tmp) throws IOException { try (VortexReader reader = VortexReader.open(file)) { Layout column = reader.layout().children().get(0); assertThat(column.isZoned()).isTrue(); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT + MemorySegment meta = column.metadata(); + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT Layout zonesFlat = column.children().get(1); SegmentSpec spec = reader.footer().segmentSpecs().get(zonesFlat.segments().getFirst()); @@ -285,8 +287,8 @@ void extensionColumn_emitsStorageMinMaxZoneMap(@TempDir Path tmp) throws IOExcep try (VortexReader reader = VortexReader.open(file)) { Layout column = reader.layout().children().get(0); assertThat(column.isZoned()).isTrue(); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT + MemorySegment meta = column.metadata(); + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT Layout zonesFlat = column.children().get(1); SegmentSpec spec = reader.footer().segmentSpecs().get(zonesFlat.segments().getFirst()); @@ -323,8 +325,8 @@ void dictColumn_emitsStringMinMaxZoneMapWrappingDict(@TempDir Path tmp) throws I Layout column = reader.layout().children().get(0); assertThat(column.isZoned()).isTrue(); assertThat(column.children().get(0).isDict()).isTrue(); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT + MemorySegment meta = column.metadata(); + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT Layout zonesFlat = column.children().get(1); SegmentSpec spec = reader.footer().segmentSpecs().get(zonesFlat.segments().getFirst()); @@ -360,8 +362,8 @@ void primitiveDictColumn_emitsNumericMinMaxZoneMapWrappingDict(@TempDir Path tmp Layout column = reader.layout().children().get(0); assertThat(column.isZoned()).isTrue(); assertThat(column.children().get(0).isDict()).isTrue(); - ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN); - assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x78); + MemorySegment meta = column.metadata(); + assertThat(meta.get(ValueLayout.JAVA_BYTE, 4)).isEqualTo((byte) 0x78); Layout zonesFlat = column.children().get(1); SegmentSpec spec = reader.footer().segmentSpecs().get(zonesFlat.segments().getFirst()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java index 88e0c4bdc..a70f4e9d4 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java @@ -10,8 +10,8 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ALPMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoALPMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.decode.AlpEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -42,11 +42,11 @@ private static DecodeContext buildAlpCtxF64( int expE, int expF, long[] encodedVals, long[] patchIndices, double[] patchValues ) { - PatchesMetadata pm = patchIndices != null - ? new PatchesMetadata((long) patchIndices.length, 0L, - io.github.dfa1.vortex.proto.PType.U32, null, null, null) + ProtoPatchesMetadata pm = patchIndices != null + ? new ProtoPatchesMetadata((long) patchIndices.length, 0L, + io.github.dfa1.vortex.proto.ProtoPType.U32, null, null, null) : null; - byte[] metaBytes = new ALPMetadata(expE, expF, pm).encode(); + byte[] metaBytes = new ProtoALPMetadata(expE, expF, pm).encode(); byte[] encBuf = new byte[encodedVals.length * 8]; ByteBuffer bb = ByteBuffer.wrap(encBuf).order(ByteOrder.LITTLE_ENDIAN); @@ -82,20 +82,20 @@ private static DecodeContext buildAlpCtxF64( } ArrayNode alpNode = ArrayNode.of(EncodingId.VORTEX_ALP, - ByteBuffer.wrap(metaBytes), children, new int[0]); + MemorySegment.ofArray(metaBytes), children, new int[0]); return new DecodeContext(alpNode, DTypes.F64, encodedVals.length, segments, REGISTRY, java.lang.foreign.Arena.global()); } private static DecodeContext buildAlpCtxF32(int expE, int expF, int[] encodedVals) { - byte[] metaBytes = new ALPMetadata(expE, expF, null).encode(); + byte[] metaBytes = new ProtoALPMetadata(expE, expF, null).encode(); byte[] encBuf = new byte[encodedVals.length * 4]; ByteBuffer bb = ByteBuffer.wrap(encBuf).order(ByteOrder.LITTLE_ENDIAN); for (int v : encodedVals) { bb.putInt(v); } ArrayNode encNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode alpNode = ArrayNode.of(EncodingId.VORTEX_ALP, ByteBuffer.wrap(metaBytes), + ArrayNode alpNode = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(metaBytes), new ArrayNode[]{encNode}, new int[0]); MemorySegment[] segments = {MemorySegment.ofArray(encBuf)}; return new DecodeContext(alpNode, DTypes.F32, encodedVals.length, segments, REGISTRY, java.lang.foreign.Arena.global()); @@ -240,8 +240,8 @@ void encode_f64_metadata_expE_isNonZero() throws Exception { EncodeResult result = ENCODER.encode(DTypes.F64, values, EncodeTestHelper.testCtx()); // Then - MemorySegment metaSeg = MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - ALPMetadata meta = ALPMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = result.rootNode().metadata(); + ProtoALPMetadata meta = ProtoALPMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.exp_e()).isGreaterThan(0); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java index f66581991..d5c23571d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ALPRDMetadata; +import io.github.dfa1.vortex.proto.ProtoALPRDMetadata; import io.github.dfa1.vortex.reader.decode.AlpRdEncodingDecoder; import io.github.dfa1.vortex.reader.decode.BitpackedEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; @@ -74,8 +74,8 @@ void encode_f64_metadata_rightBitWidth_isNonZero() throws Exception { // When EncodeResult result = sut.encode(DTypes.F64, values, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - ALPRDMetadata meta = ALPRDMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoALPRDMetadata meta = ProtoALPRDMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.right_bit_width()).isGreaterThan(0); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java index 8c607c88a..72b0f1132 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java @@ -7,16 +7,14 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; -import io.github.dfa1.vortex.proto.BitPackedMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.decode.BitpackedEncodingDecoder; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import static org.assertj.core.api.Assertions.assertThat; @@ -34,16 +32,16 @@ void bitpackedDecode_withConstantPatchesValues_broadcastsValueAcrossPatches() { byte[] packed = new byte[128]; - ScalarValue idxScalar = ScalarValue.ofUint64Value(2L); + ProtoScalarValue idxScalar = ProtoScalarValue.ofUint64Value(2L); byte[] idxScalarBytes = idxScalar.encode(); - ScalarValue valScalar = ScalarValue.ofInt64Value(constantPatchValue); + ProtoScalarValue valScalar = ProtoScalarValue.ofInt64Value(constantPatchValue); byte[] valScalarBytes = valScalar.encode(); - PatchesMetadata patches = new PatchesMetadata(numPatches, 0, - io.github.dfa1.vortex.proto.PType.U32, null, null, null); - BitPackedMetadata meta = new BitPackedMetadata(1, 0, patches); - ByteBuffer metaBuf = ByteBuffer.wrap(meta.encode()).order(ByteOrder.LITTLE_ENDIAN); + ProtoPatchesMetadata patches = new ProtoPatchesMetadata(numPatches, 0, + io.github.dfa1.vortex.proto.ProtoPType.U32, null, null, null); + ProtoBitPackedMetadata meta = new ProtoBitPackedMetadata(1, 0, patches); + MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); try (Arena arena = Arena.ofConfined()) { MemorySegment packedSeg = arena.allocate(packed.length, 8); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java index afd8038a4..4fa451f99 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.BitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.reader.decode.BitpackedEncodingDecoder; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -79,8 +79,8 @@ void encode_i32_metadata_bitWidth_isNonZero() throws Exception { int[] data = {1, 2, 3, 4, 5}; EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - BitPackedMetadata meta = BitPackedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoBitPackedMetadata meta = ProtoBitPackedMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.bit_width()).isGreaterThan(0); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java index 510b320cc..d3eb3bed3 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java @@ -9,8 +9,8 @@ import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.BitPackedMetadata; -import io.github.dfa1.vortex.proto.PatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.decode.BitpackedEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -41,9 +41,9 @@ void decode_appliesPatches_overridingBitPackedValues() { MemorySegment packedSeg = packed.buffers().getFirst(); byte[] packedBytes = packedSeg.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE); - PatchesMetadata patches = new PatchesMetadata(2L, 0L, - io.github.dfa1.vortex.proto.PType.U32, null, null, null); - byte[] metaBytes = new BitPackedMetadata(6, 0, patches).encode(); + ProtoPatchesMetadata patches = new ProtoPatchesMetadata(2L, 0L, + io.github.dfa1.vortex.proto.ProtoPType.U32, null, null, null); + byte[] metaBytes = new ProtoBitPackedMetadata(6, 0, patches).encode(); byte[] idxBuf = new byte[2 * 4]; ByteBuffer.wrap(idxBuf).order(ByteOrder.LITTLE_ENDIAN).putInt(1).putInt(3); @@ -55,7 +55,7 @@ void decode_appliesPatches_overridingBitPackedValues() { ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); ArrayNode bpNode = ArrayNode.of(EncodingId.FASTLANES_BITPACKED, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new ArrayNode[]{idxNode, valNode}, new int[]{0}); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java index f90eb340d..03fb8b65d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.encoding.TimeUnit; -import io.github.dfa1.vortex.proto.DateTimePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDateTimePartsMetadata; import io.github.dfa1.vortex.reader.decode.DateTimePartsEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -20,8 +20,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -36,10 +34,7 @@ class DateTimePartsEncodingEncoderTest { private static final DType EXT_TIMESTAMP_NS = timestampDType(TimeUnit.Nanoseconds); private static DType timestampDType(TimeUnit unit) { - ByteBuffer meta = ByteBuffer.allocate(3).order(ByteOrder.LITTLE_ENDIAN); - meta.put((byte) unit.ordinal()); - meta.putShort((short) 0); - meta.flip(); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) unit.ordinal(), 0, 0}); return new DType.Extension("vortex.timestamp", DType.I64, meta, false); } @@ -212,8 +207,8 @@ void encode_metadata_ptypes_areI64() throws Exception { EncodeResult result = ENCODER.encode(EXT_TIMESTAMP_MS, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - DateTimePartsMetadata meta = DateTimePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoDateTimePartsMetadata meta = ProtoDateTimePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.days_ptype().value()).isEqualTo(7); assertThat(meta.seconds_ptype().value()).isEqualTo(7); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoderTest.java index 5691aadb7..12cfba715 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalBytePartsEncodingEncoderTest.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.DecimalBytePartsMetadata; +import io.github.dfa1.vortex.proto.ProtoDecimalBytePartsMetadata; import io.github.dfa1.vortex.reader.decode.DecimalBytePartsEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Test; @@ -67,10 +67,9 @@ void metadata_zerothChildPtype_isI64_lowerPartCountIsZero() throws Exception { EncodeResult result = sut.encode(dtype, values, EncodeTestHelper.testCtx()); // Then - byte[] metaBytes = new byte[result.rootNode().metadata().remaining()]; - result.rootNode().metadata().duplicate().get(metaBytes); - DecimalBytePartsMetadata meta = - DecimalBytePartsMetadata.decode(java.lang.foreign.MemorySegment.ofArray(metaBytes), 0, metaBytes.length); + java.lang.foreign.MemorySegment metaSeg = result.rootNode().metadata(); + ProtoDecimalBytePartsMetadata meta = + ProtoDecimalBytePartsMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.zeroth_child_ptype().value()).isEqualTo(7); // I64 ordinal assertThat(meta.lower_part_count()).isZero(); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java index 513ed0beb..4224703dd 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java @@ -9,7 +9,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.encoding.TestSegments; -import io.github.dfa1.vortex.proto.DecimalMetadata; +import io.github.dfa1.vortex.proto.ProtoDecimalMetadata; import io.github.dfa1.vortex.reader.decode.DecimalEncodingDecoder; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -85,9 +85,8 @@ void valuesType_matchesPrecisionBoundaries(int precision, int expectedValuesType EncodeResult encoded = ENCODER.encode(dtype, input, EncodeTestHelper.testCtx()); // Then - byte[] metaBytes = new byte[encoded.rootNode().metadata().remaining()]; - encoded.rootNode().metadata().duplicate().get(metaBytes); - DecimalMetadata meta = DecimalMetadata.decode(java.lang.foreign.MemorySegment.ofArray(metaBytes), 0, metaBytes.length); + java.lang.foreign.MemorySegment metaSeg = encoded.rootNode().metadata(); + ProtoDecimalMetadata meta = ProtoDecimalMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.values_type()).isEqualTo(expectedValuesType); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java index a5862c148..f6d357324 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java @@ -9,8 +9,8 @@ import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.DeltaMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoDeltaMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.decode.DeltaEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Test; @@ -122,8 +122,8 @@ void encode_i64_metadata_deltasLen_isNonZero() throws Exception { // When EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - MemorySegment metaSeg = MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - DeltaMetadata meta = DeltaMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = result.rootNode().metadata(); + ProtoDeltaMetadata meta = ProtoDeltaMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.deltas_len()).isGreaterThan(0); @@ -212,9 +212,9 @@ void encode_empty_hasNoStats() { assertThat(result.hasStats()).isFalse(); } - private static ScalarValue scalar(byte[] bytes) throws java.io.IOException { + private static ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { MemorySegment seg = MemorySegment.ofArray(bytes); - return ScalarValue.decode(seg, 0, seg.byteSize()); + return ProtoScalarValue.decode(seg, 0, seg.byteSize()); } private static Stream randomIntegerArrays() { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java index d358a2cef..adaacb68b 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.DictMetadata; +import io.github.dfa1.vortex.proto.ProtoDictMetadata; import io.github.dfa1.vortex.reader.decode.DictEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import io.github.dfa1.vortex.reader.decode.VarBinEncodingDecoder; @@ -172,8 +172,8 @@ void encode_utf8_metadata_valuesLen_matchesUniqueCount() throws Exception { // When EncodeResult result = ENCODER.encode(DTypes.UTF8, data, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - DictMetadata meta = DictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoDictMetadata meta = ProtoDictMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.values_len()).isEqualTo(2); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java index b0209869c..0e0780147 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java @@ -13,7 +13,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ScalarValue; +import io.github.dfa1.vortex.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder; import io.github.dfa1.vortex.reader.decode.FrameOfReferenceEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; @@ -42,7 +42,7 @@ class Decode { private static DecodeContext buildForContext( DType dtype, long reference, long[] residuals, PType ptype ) { - byte[] metaBytes = ScalarValue.ofInt64Value(reference).encode(); + byte[] metaBytes = ProtoScalarValue.ofInt64Value(reference).encode(); int elemBytes = ptype.byteSize(); byte[] childBytes = new byte[residuals.length * elemBytes]; @@ -58,7 +58,7 @@ private static DecodeContext buildForContext( ArrayNode childNode = ArrayNode.of( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode forNode = ArrayNode.of( - EncodingId.FASTLANES_FOR, ByteBuffer.wrap(metaBytes), + EncodingId.FASTLANES_FOR, MemorySegment.ofArray(metaBytes), new ArrayNode[]{childNode}, new int[0]); MemorySegment[] segments = {MemorySegment.ofArray(childBytes)}; @@ -150,9 +150,9 @@ void decode_nullableResiduals_returnsMaskedArrayWithCorrectValues() { EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); ArrayNode primNode = ArrayNode.of( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[]{validityNode}, new int[]{0}); - byte[] metaBytes = ScalarValue.ofInt64Value(reference).encode(); + byte[] metaBytes = ProtoScalarValue.ofInt64Value(reference).encode(); ArrayNode forNode = ArrayNode.of( - EncodingId.FASTLANES_FOR, ByteBuffer.wrap(metaBytes), new ArrayNode[]{primNode}, new int[0]); + EncodingId.FASTLANES_FOR, MemorySegment.ofArray(metaBytes), new ArrayNode[]{primNode}, new int[0]); ReadRegistry registry = TestRegistry.ofDecoders( new FrameOfReferenceEncodingDecoder(), new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java index 8ba55aa7d..73d437117 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java @@ -11,7 +11,7 @@ import io.github.dfa1.vortex.encoding.PTypeIO; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.FSSTMetadata; +import io.github.dfa1.vortex.proto.ProtoFSSTMetadata; import io.github.dfa1.vortex.reader.decode.FsstEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -23,7 +23,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.stream.Stream; @@ -155,14 +154,14 @@ private static DecodeContext buildCtx( MemorySegment[] segs = {symBuf, symLenBuf, compBuf, uncompLenBuf, codesOffBuf}; - byte[] metaBytes = new FSSTMetadata(io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal()), io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal())).encode(); + byte[] metaBytes = new ProtoFSSTMetadata(io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal()), io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal())).encode(); ArrayNode uncompLensNode = ArrayNode.of( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); ArrayNode codesOffNode = ArrayNode.of( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{4}); ArrayNode root = ArrayNode.of( - EncodingId.VORTEX_FSST, ByteBuffer.wrap(metaBytes), + EncodingId.VORTEX_FSST, MemorySegment.ofArray(metaBytes), new ArrayNode[]{uncompLensNode, codesOffNode}, new int[]{0, 1, 2}); return new DecodeContext(root, DTypes.UTF8, n, segs, REGISTRY, arena); @@ -249,8 +248,8 @@ void encode_metadata_ptypes_areI32() throws Exception { // When EncodeResult result = ENCODER.encode(DTypes.UTF8, data, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - FSSTMetadata meta = FSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoFSSTMetadata meta = ProtoFSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.uncompressed_lengths_ptype().value()).isEqualTo(6); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java index ecef30b83..33a1378db 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ListMetadata; +import io.github.dfa1.vortex.proto.ProtoListMetadata; import io.github.dfa1.vortex.reader.decode.ListEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -167,7 +167,7 @@ void decode_wrongChildCount_throws() { ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[0]); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LIST, - java.nio.ByteBuffer.wrap(new byte[0]), + MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.LIST_I32, 0, new MemorySegment[0], REGISTRY, Arena.global()); @@ -190,8 +190,8 @@ void encode_metadata_elementsLen_matchesElementCount() throws Exception { EncodeResult result = ENCODER.encode(DTypes.LIST_I32, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - ListMetadata meta = ListMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoListMetadata meta = ProtoListMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.elements_len()).isEqualTo(5); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java index 0b323bb05..f4c93a21a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestDecodeContexts; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ListViewMetadata; +import io.github.dfa1.vortex.proto.ProtoListViewMetadata; import io.github.dfa1.vortex.reader.decode.ListViewEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -163,7 +163,7 @@ void decode_wrongChildCount_throws() { ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[0]); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LISTVIEW, - java.nio.ByteBuffer.wrap(new byte[0]), + MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = TestDecodeContexts.of(node, DTypes.LIST_I32).registry(REGISTRY).build(); @@ -185,9 +185,8 @@ void encode_metadata_elementsLen_matchesElementCount() throws Exception { // When EncodeResult result = ENCODER.encode(DTypes.LIST_I32, data, EncodeTestHelper.testCtx()); - java.nio.ByteBuffer metaBuf = result.rootNode().metadata().duplicate(); - java.lang.foreign.MemorySegment metaSeg = java.lang.foreign.MemorySegment.ofBuffer(metaBuf); - ListViewMetadata meta = ListViewMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + java.lang.foreign.MemorySegment metaSeg = result.rootNode().metadata(); + ProtoListViewMetadata meta = ProtoListViewMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.elements_len()).isEqualTo(5); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java index fb99fd16b..78035d943 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java @@ -12,7 +12,7 @@ import io.github.dfa1.vortex.reader.decode.KnownArrayNode; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.RLEMetadata; +import io.github.dfa1.vortex.proto.ProtoRLEMetadata; import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import io.github.dfa1.vortex.reader.decode.RleEncodingDecoder; @@ -329,8 +329,8 @@ void encode_i32_metadata_valuesLen_matchesRunCount() throws Exception { EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - RLEMetadata meta = RLEMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoRLEMetadata meta = ProtoRLEMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.values_len()).isEqualTo(2); assertThat(meta.indices_len()).isGreaterThan(0); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java index cfa6528b9..d3ce1829c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.RunEndMetadata; +import io.github.dfa1.vortex.proto.ProtoRunEndMetadata; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import io.github.dfa1.vortex.reader.decode.RunEndEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -38,8 +38,8 @@ private static DecodeContext buildCtx( DType dtype, long rowCount, long[] ends, long[] values, PType endsPtype, long offset ) { - byte[] metaBytes = new RunEndMetadata( - io.github.dfa1.vortex.proto.PType.fromValue(endsPtype.ordinal()), ends.length, offset).encode(); + byte[] metaBytes = new ProtoRunEndMetadata( + io.github.dfa1.vortex.proto.ProtoPType.fromValue(endsPtype.ordinal()), ends.length, offset).encode(); byte[] endsBuf = toLEBytes(ends, endsPtype); byte[] valBuf = toLEBytes(values, PType.I64); @@ -49,7 +49,7 @@ private static DecodeContext buildCtx( ArrayNode valsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode reNode = ArrayNode.of(EncodingId.VORTEX_RUNEND, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new ArrayNode[]{endsNode, valsNode}, new int[0]); @@ -168,8 +168,8 @@ void encode_i64_metadata_numRuns_andEndsPtype() throws Exception { // When EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - RunEndMetadata meta = RunEndMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoRunEndMetadata meta = ProtoRunEndMetadata.decode(metaSeg, 0, metaSeg.byteSize()); // Then assertThat(meta.num_runs()).isEqualTo(3); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index 4071e5d46..46c2503e8 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -16,8 +16,8 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SequenceMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSequenceMetadata; import io.github.dfa1.vortex.reader.decode.SequenceEncodingDecoder; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -27,7 +27,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +41,7 @@ class SequenceEncodingEncoderTest { class Encode { private static DecodeContext encodeResultToCtx(EncodeResult result, DType dtype, long n) { - ByteBuffer meta = result.rootNode().metadata(); + MemorySegment meta = result.rootNode().metadata(); ArrayNode node = ArrayNode.of(EncodingId.VORTEX_SEQUENCE, meta, new ArrayNode[0], new int[0]); return new DecodeContext(node, dtype, n, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); } @@ -120,8 +119,8 @@ void encode_u64_roundTrips_andMetadataIsUnsigned() throws Exception { // When EncodeResult resultEncoded = ENCODER.encode(DTypes.U64, data, EncodeTestHelper.testCtx()); - MemorySegment metaSeg = MemorySegment.ofBuffer(resultEncoded.rootNode().metadata().duplicate()); - SequenceMetadata meta = SequenceMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = resultEncoded.rootNode().metadata(); + ProtoSequenceMetadata meta = ProtoSequenceMetadata.decode(metaSeg, 0, metaSeg.byteSize()); DecodeContext ctx = encodeResultToCtx(resultEncoded, DTypes.U64, data.length); LongArray result = (LongArray) DECODER.decode(ctx); @@ -316,26 +315,26 @@ static Stream i32Sequences() { private static DecodeContext makeCtx(byte[] meta, DType dtype, long n) { ArrayNode node = ArrayNode.of(EncodingId.VORTEX_SEQUENCE, - ByteBuffer.wrap(meta), new ArrayNode[0], new int[0]); + MemorySegment.ofArray(meta), new ArrayNode[0], new int[0]); return new DecodeContext(node, dtype, n, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); } private static byte[] intMeta(long base, long mul) { - return new SequenceMetadata(ScalarValue.ofInt64Value(base), ScalarValue.ofInt64Value(mul)).encode(); + return new ProtoSequenceMetadata(ProtoScalarValue.ofInt64Value(base), ProtoScalarValue.ofInt64Value(mul)).encode(); } private static byte[] f64Meta(double base, double mul) { - return new SequenceMetadata(ScalarValue.ofF64Value(base), ScalarValue.ofF64Value(mul)).encode(); + return new ProtoSequenceMetadata(ProtoScalarValue.ofF64Value(base), ProtoScalarValue.ofF64Value(mul)).encode(); } private static byte[] f32Meta(float base, float mul) { - return new SequenceMetadata(ScalarValue.ofF32Value(base), ScalarValue.ofF32Value(mul)).encode(); + return new ProtoSequenceMetadata(ProtoScalarValue.ofF32Value(base), ProtoScalarValue.ofF32Value(mul)).encode(); } private static byte[] f16Meta(short baseShort, short mulShort) { - return new SequenceMetadata( - ScalarValue.ofF16Value(Short.toUnsignedLong(baseShort)), - ScalarValue.ofF16Value(Short.toUnsignedLong(mulShort))).encode(); + return new ProtoSequenceMetadata( + ProtoScalarValue.ofF16Value(Short.toUnsignedLong(baseShort)), + ProtoScalarValue.ofF16Value(Short.toUnsignedLong(mulShort))).encode(); } @ParameterizedTest @@ -438,8 +437,8 @@ class Metadata { void encode_i64_metadata_base_andMultiplier_areSet() throws Exception { long[] data = {10L, 12L, 14L, 16L}; EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - MemorySegment metaSeg = MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - SequenceMetadata meta = SequenceMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + MemorySegment metaSeg = result.rootNode().metadata(); + ProtoSequenceMetadata meta = ProtoSequenceMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.base()).isNotNull(); assertThat(meta.multiplier()).isNotNull(); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java index b4a9ecc43..44d32e942 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java @@ -14,11 +14,11 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.NullValue; -import io.github.dfa1.vortex.proto.PatchesMetadata; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.SparseMetadata; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoNullValue; +import io.github.dfa1.vortex.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoSparseMetadata; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import io.github.dfa1.vortex.reader.decode.SparseEncodingDecoder; @@ -72,8 +72,8 @@ void encode_allZeros_noPatches() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - SparseMetadata meta = SparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoSparseMetadata meta = ProtoSparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.patches().len()).isZero(); } @@ -86,8 +86,8 @@ void encode_withNonZero_createsPatches() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - SparseMetadata meta = SparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoSparseMetadata meta = ProtoSparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.patches().len()).isEqualTo(2); } @@ -133,8 +133,8 @@ void encode_empty_or_allZero_noPatches(int size) throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - SparseMetadata meta = SparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoSparseMetadata meta = ProtoSparseMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.patches().len()).isZero(); } } @@ -153,7 +153,7 @@ private static DecodeContext buildSparseCtxWithOffset( DType dtype, long rowCount, long fillLong, PType idxPtype, long[] patchIndices, long[] patchValues, long offset ) { - byte[] fillBytes = ScalarValue.ofInt64Value(fillLong).encode(); + byte[] fillBytes = ProtoScalarValue.ofInt64Value(fillLong).encode(); byte[] metaBytes = buildSparseMetaBytes(patchIndices.length, offset, idxPtype); byte[] idxBuf = toLEBytes(patchIndices, idxPtype); byte[] valBuf = toLEBytes(patchValues, PType.I64); @@ -165,7 +165,7 @@ private static DecodeContext buildSparseCtxF64( DType dtype, long rowCount, double fillDouble, long[] patchIndices, double[] patchValues ) { - byte[] fillBytes = ScalarValue.ofF64Value(fillDouble).encode(); + byte[] fillBytes = ProtoScalarValue.ofF64Value(fillDouble).encode(); byte[] metaBytes = buildSparseMetaBytes(patchIndices.length, 0L, PType.U32); byte[] idxBuf = toLEBytes(patchIndices, PType.U32); byte[] valBuf = f64LEBytes(patchValues); @@ -180,7 +180,7 @@ private static DecodeContext buildCtx(DType dtype, long rowCount, ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, - ByteBuffer.wrap(metaBytes), + MemorySegment.ofArray(metaBytes), new ArrayNode[]{idxNode, valNode}, new int[]{0}); @@ -193,9 +193,9 @@ private static DecodeContext buildCtx(DType dtype, long rowCount, } private static byte[] buildSparseMetaBytes(long numPatches, long offset, PType idxPtype) { - PatchesMetadata patchesMeta = new PatchesMetadata(numPatches, offset, - io.github.dfa1.vortex.proto.PType.fromValue(idxPtype.ordinal()), null, null, null); - return new SparseMetadata(patchesMeta).encode(); + ProtoPatchesMetadata patchesMeta = new ProtoPatchesMetadata(numPatches, offset, + io.github.dfa1.vortex.proto.ProtoPType.fromValue(idxPtype.ordinal()), null, null, null); + return new ProtoSparseMetadata(patchesMeta).encode(); } private static byte[] toLEBytes(long[] values, PType ptype) { @@ -303,7 +303,7 @@ void decode_offsetSubtracted() { @Test void decode_nullValueFill_treatedAsZero() { // Given - byte[] nullFill = ScalarValue.ofNullValue(NullValue.NULL_VALUE).encode(); + byte[] nullFill = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE).encode(); byte[] meta = buildSparseMetaBytes(0, 0L, PType.U32); DecodeContext ctx = buildCtx(DTypes.I64, 4, nullFill, meta, new byte[0], new byte[0]); @@ -321,7 +321,7 @@ void decode_nullValueFill_treatedAsZero() { void decode_utf8_noPatches_allEmpty() { // Given DType utf8 = new DType.Utf8(true); - byte[] nullFill = ScalarValue.ofNullValue(NullValue.NULL_VALUE).encode(); + byte[] nullFill = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE).encode(); byte[] meta = buildSparseMetaBytes(0, 0L, PType.U32); DecodeContext ctx = buildCtx(utf8, 3, nullFill, meta, new byte[0], new byte[0]); @@ -340,23 +340,23 @@ void decode_utf8_noPatches_allEmpty() { void decode_utf8_withPatches_writesStringsAtIndices() { // Given DType utf8 = new DType.Utf8(true); - byte[] nullFill = ScalarValue.ofNullValue(NullValue.NULL_VALUE).encode(); + byte[] nullFill = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE).encode(); byte[] meta = buildSparseMetaBytes(2, 0L, PType.U32); byte[] idxBuf = toLEBytes(new long[]{1L, 3L}, PType.U32); byte[] strBytes = "hibye".getBytes(StandardCharsets.UTF_8); byte[] offsets = intLEBytes(new int[]{0, 2, 5}); - byte[] varBinMeta = new VarBinMetadata(io.github.dfa1.vortex.proto.PType.fromValue(PType.I32.ordinal())).encode(); + byte[] varBinMeta = new ProtoVarBinMetadata(io.github.dfa1.vortex.proto.ProtoPType.fromValue(PType.I32.ordinal())).encode(); ArrayNode offsetsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_VARBIN, - ByteBuffer.wrap(varBinMeta), + MemorySegment.ofArray(varBinMeta), new ArrayNode[]{offsetsNode}, new int[]{2}); ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, - ByteBuffer.wrap(meta), + MemorySegment.ofArray(meta), new ArrayNode[]{idxNode, valNode}, new int[]{0}); ReadRegistry registry = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder(), new VarBinEncodingDecoder()); @@ -386,7 +386,7 @@ void decode_utf8_withPatches_writesStringsAtIndices() { void decode_bool_withPatches_setsBitsAtIndices() { // Given DType bool = new DType.Bool(true); - byte[] nullFill = ScalarValue.ofNullValue(NullValue.NULL_VALUE).encode(); + byte[] nullFill = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE).encode(); byte[] meta = buildSparseMetaBytes(2, 0L, PType.U32); byte[] idxBuf = toLEBytes(new long[]{2L, 5L}, PType.U32); byte[] boolBits = new byte[]{0b00000011}; @@ -396,7 +396,7 @@ void decode_bool_withPatches_setsBitsAtIndices() { ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, - ByteBuffer.wrap(meta), + MemorySegment.ofArray(meta), new ArrayNode[]{idxNode, valNode}, new int[]{0}); ReadRegistry registry = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java index 7145234b0..bf8420bec 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.extension.TimeDtype; import org.junit.jupiter.api.Test; -import java.nio.ByteBuffer; +import java.lang.foreign.MemorySegment; import java.time.LocalTime; import java.util.Arrays; import java.util.List; @@ -154,8 +154,7 @@ void encodeAll_nullInNonNullableColumn_throws() { @Test void encodeAll_daysUnit_throws() { // Given a hand-built Days-tagged dtype (TimeDtype.of rejects Days, so build directly) - ByteBuffer meta = ByteBuffer.allocate(1); - meta.put(0, (byte) TimeUnit.Days.ordinal()); + MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) TimeUnit.Days.ordinal()}); DType.Extension dtype = new DType.Extension( ExtensionId.VORTEX_TIME.id(), DType.I32, meta, false); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java index 82c1df76c..712b3f41f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.VarBinMetadata; +import io.github.dfa1.vortex.proto.ProtoVarBinMetadata; import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder; import io.github.dfa1.vortex.reader.decode.VarBinEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -182,8 +182,8 @@ void encode_utf8_metadata_offsetsPtype_isI64() throws Exception { EncodeResult result = ENCODER.encode(DTypes.UTF8, data, EncodeTestHelper.testCtx()); // Then - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - VarBinMetadata meta = VarBinMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoVarBinMetadata meta = ProtoVarBinMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.offsets_ptype().value()).isEqualTo(7); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java index fff9ce8da..6dccc2ea8 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java @@ -3,10 +3,10 @@ import io.github.dfa1.vortex.core.DType; import io.github.dfa1.vortex.core.VortexException; import io.github.dfa1.vortex.encoding.EncodingId; -import io.github.dfa1.vortex.proto.Primitive; -import io.github.dfa1.vortex.proto.Scalar; -import io.github.dfa1.vortex.proto.ScalarValue; -import io.github.dfa1.vortex.proto.VariantMetadata; +import io.github.dfa1.vortex.proto.ProtoPrimitive; +import io.github.dfa1.vortex.proto.ProtoScalar; +import io.github.dfa1.vortex.proto.ProtoScalarValue; +import io.github.dfa1.vortex.proto.ProtoVariantMetadata; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -22,17 +22,17 @@ class VariantEncodingEncoderTest { private static final VariantEncodingEncoder SUT = new VariantEncodingEncoder(); private static final DType.Variant VARIANT = DType.VARIANT; - private static Scalar i32Scalar(long value) { + private static ProtoScalar i32Scalar(long value) { // Inner typed scalar carrying its own i32 dtype, wrapped as a variant value - // (mirrors Rust Scalar::variant(Scalar::primitive(value))). - return new Scalar( - io.github.dfa1.vortex.proto.DType.ofPrimitive( - new Primitive(io.github.dfa1.vortex.proto.PType.I32, false)), - ScalarValue.ofInt64Value(value)); + // (mirrors Rust ProtoScalar::variant(ProtoScalar::primitive(value))). + return new ProtoScalar( + io.github.dfa1.vortex.proto.ProtoDType.ofPrimitive( + new ProtoPrimitive(io.github.dfa1.vortex.proto.ProtoPType.I32, false)), + ProtoScalarValue.ofInt64Value(value)); } private static long innerInt(MemorySegment buf) throws Exception { - ScalarValue scalar = ScalarValue.decode(buf, 0, buf.byteSize()); + ProtoScalarValue scalar = ProtoScalarValue.decode(buf, 0, buf.byteSize()); assertThat(scalar.variant_value()).isNotNull(); return scalar.variant_value().value().int64_value(); } @@ -75,8 +75,8 @@ void allEqual_emitsSingleConstantChild() throws Exception { void metadataHasNoShreddedDtype() throws Exception { EncodeResult result = SUT.encode(VARIANT, VariantData.constant(3, i32Scalar(7L)), EncodeTestHelper.testCtx()); - MemorySegment meta = MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - VariantMetadata decoded = VariantMetadata.decode(meta, 0, meta.byteSize()); + MemorySegment meta = result.rootNode().metadata(); + ProtoVariantMetadata decoded = ProtoVariantMetadata.decode(meta, 0, meta.byteSize()); assertThat(decoded.shredded_dtype()).isNull(); } } @@ -234,11 +234,11 @@ void emitsSecondChildAndRecordsShreddedDtype() throws Exception { assertThat(root.children()[1].encodingId()).isEqualTo(EncodingId.VORTEX_PRIMITIVE); // ...and the metadata records shredded_dtype = i32. - MemorySegment meta = MemorySegment.ofBuffer(root.metadata().duplicate()); - VariantMetadata vm = VariantMetadata.decode(meta, 0, meta.byteSize()); + MemorySegment meta = root.metadata(); + ProtoVariantMetadata vm = ProtoVariantMetadata.decode(meta, 0, meta.byteSize()); assertThat(vm.shredded_dtype()).isNotNull(); assertThat(vm.shredded_dtype().primitive()).isNotNull(); - assertThat(vm.shredded_dtype().primitive().type()).isEqualTo(io.github.dfa1.vortex.proto.PType.I32); + assertThat(vm.shredded_dtype().primitive().type()).isEqualTo(io.github.dfa1.vortex.proto.ProtoPType.I32); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java index cc3eedb16..d12aa3f65 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java @@ -16,8 +16,8 @@ import io.github.dfa1.vortex.encoding.EncodingId; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; -import io.github.dfa1.vortex.proto.ZstdFrameMetadata; -import io.github.dfa1.vortex.proto.ZstdMetadata; +import io.github.dfa1.vortex.proto.ProtoZstdFrameMetadata; +import io.github.dfa1.vortex.proto.ProtoZstdMetadata; import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder; import io.github.dfa1.vortex.reader.decode.ZstdEncodingDecoder; import org.junit.jupiter.api.Nested; @@ -107,7 +107,7 @@ private static DecodeContext makeDictCtx( segments[i + 1] = MemorySegment.ofArray(compressedFrames[i]); bufIndices[i + 1] = i + 1; } - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, ByteBuffer.wrap(meta), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[0], bufIndices); return new DecodeContext(node, dtype, n, segments, ReadRegistry.empty(), Arena.ofAuto()); } @@ -128,7 +128,7 @@ private static DecodeContext makeNullableCtx( allSegments.addAll(validityResult.buffers()); ArrayNode validityNode = toArrayNode(remappedValidity); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, ByteBuffer.wrap(meta), + ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[]{validityNode}, bufIndices); ReadRegistry registry = TestRegistry.ofDecoders(new BoolEncodingDecoder()); @@ -153,11 +153,11 @@ private static byte[] compress(byte[] input) { } private static byte[] metaNoDict(long[] uncompressedSizes, long[] nValues) { - java.util.List frames = new java.util.ArrayList<>(); + java.util.List frames = new java.util.ArrayList<>(); for (int i = 0; i < uncompressedSizes.length; i++) { - frames.add(new ZstdFrameMetadata(uncompressedSizes[i], nValues[i])); + frames.add(new ProtoZstdFrameMetadata(uncompressedSizes[i], nValues[i])); } - return new ZstdMetadata(0, frames).encode(); + return new ProtoZstdMetadata(0, frames).encode(); } private static byte[] toLeBytes(int[] values) { @@ -187,8 +187,8 @@ void decode_withDictionary_throws() { // Given — metadata with non-zero dictionary_size; pure-Java decoder doesn't support // dictionary-compressed Zstd (no JNI dependency) byte[] compressed = compress(toLeBytes(new int[]{1, 2, 3})); - byte[] meta = new ZstdMetadata(256, - java.util.List.of(new ZstdFrameMetadata(12, 3))).encode(); + byte[] meta = new ProtoZstdMetadata(256, + java.util.List.of(new ProtoZstdFrameMetadata(12, 3))).encode(); DecodeContext ctx = makeDictCtx(meta, DTypes.I32, 3, new byte[256], compressed); // When / Then @@ -280,8 +280,8 @@ void encode_i32_metadata_framesCount_isNonZero() throws Exception { data[i] = i; } EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); - var metaSeg = java.lang.foreign.MemorySegment.ofBuffer(result.rootNode().metadata().duplicate()); - ZstdMetadata meta = ZstdMetadata.decode(metaSeg, 0, metaSeg.byteSize()); + var metaSeg = result.rootNode().metadata(); + ProtoZstdMetadata meta = ProtoZstdMetadata.decode(metaSeg, 0, metaSeg.byteSize()); assertThat(meta.frames()).isNotEmpty(); }