diff --git a/bindings/go/vcencrypt/README.md b/bindings/go/vcencrypt/README.md index d1ff1063..24b6af14 100644 --- a/bindings/go/vcencrypt/README.md +++ b/bindings/go/vcencrypt/README.md @@ -231,7 +231,8 @@ these find inputs where both are self-consistent but differ from each other. - **`FuzzDifferentialValue`** checks that both sides accept or reject the same value bytes. When both accept, it checks that Rust re-encodes the - input exactly and that Go's re-encoding matches, allowing only Go's + input exactly, apart from a typed null, which both sides write back as + the plain null, and that Go's re-encoding matches, allowing only Go's `undefined` to `null` projection. - **`FuzzDifferentialCipherText`** checks that both sides accept or reject the same ciphertext bytes. Fuzzed seals never authenticate, so there is no diff --git a/bindings/go/vcencrypt/differential_fuzz_test.go b/bindings/go/vcencrypt/differential_fuzz_test.go index bda8e61e..34dba19e 100644 --- a/bindings/go/vcencrypt/differential_fuzz_test.go +++ b/bindings/go/vcencrypt/differential_fuzz_test.go @@ -129,6 +129,21 @@ func corpusCiphertexts(f *testing.F) [][]byte { return out } +// intWidths is the payload width of each int tag, which a timestamp in the +// envelope rides on. +var intWidths = map[byte]int{0x0C: 1, 0x0D: 1, 0x0E: 2, 0x0F: 2, 0x04: 4, 0x06: 4, 0x05: 8, 0x07: 8, 0x10: 16, 0x11: 16} + +// typedNullWidths is the payload width of each header a typed null may +// carry: an int, a float, a date, a timestamp, a decimal, and bytes, which +// only a UUID's typed null carries, sixteen wide. +var typedNullWidths = func() map[byte]int { + m := map[byte]int{0x08: 4, 0x12: 4, 0x09: 8, 0x13: 12, 0x14: 16, 0x0B: 16} + for tag, width := range intWidths { + m[tag] = width + } + return m +}() + // requireGoMatchesRust fails unless goOut is rustOut with exactly Go's // documented projections applied: an Undefined value-node tag (0x01) turned // into Null (0x00), and a timestamp in the 0xF3 envelope (seconds, @@ -144,8 +159,9 @@ func requireGoMatchesRust(t *testing.T, data, rustOut, goOut []byte) { } // rustCanonical is what Rust re-encodes an accepted value as: the input, -// with a decimal negative zero read back as zero (ADR 0001). Rust keeps -// Undefined and envelope timestamps as they are. +// with a decimal negative zero read back as zero (ADR 0001) and a typed null +// written as the plain null, since a frame is never sealed (vitaminc#493). +// Rust keeps Undefined and envelope timestamps as they are. func rustCanonical(data []byte) []byte { return rewrite(data, false) } @@ -153,7 +169,8 @@ func rustCanonical(data []byte) []byte { // rewrite walks an accepted value tree following the transport format // independently of either decoder, so a walk that falls out of step with it // produces the wrong bytes and fails the comparison rather than passing. It -// always normalizes a decimal negative zero; with goProjection it also +// always normalizes a decimal negative zero and writes a typed null as the +// plain null; with goProjection it also // applies Go's two projections. vcffi's fuzz tests have the same walker, but // this is a separate Go module and cannot import test code. func rewrite(data []byte, goProjection bool) []byte { @@ -226,6 +243,18 @@ func rewrite(data []byte, goProjection bool) []byte { case 0xF2: // Passthrough out = append(out, tag) walk() + case 0x15: // a typed null: both sides write it back as the plain null + header := take(1) + if len(header) == 1 && header[0] == 0xF3 { + // id and parameter, then the physical tag the width follows. + if envelope := take(3); len(envelope) == 3 { + header = envelope[2:] + } + } + if len(header) == 1 { + take(typedNullWidths[header[0]] - 1) + } + out = append(out, 0x00) case 0xF3: // the envelope: id, parameter, then a physical scalar header := take(2) if goProjection && len(header) == 2 && header[0] == 3 && header[1] <= 2 { @@ -253,8 +282,7 @@ func shorthandTimestamp(data []byte, pos *int, unit byte) ([]byte, bool) { return nil, false } tag := data[*pos] - widths := map[byte]int{0x0C: 1, 0x0D: 1, 0x0E: 2, 0x0F: 2, 0x04: 4, 0x06: 4, 0x05: 8, 0x07: 8, 0x10: 16, 0x11: 16} - width, ok := widths[tag] + width, ok := intWidths[tag] if !ok || *pos+1+width > len(data) { return nil, false } diff --git a/bindings/go/vcencrypt/typed_null_test.go b/bindings/go/vcencrypt/typed_null_test.go new file mode 100644 index 00000000..e2c69d8d --- /dev/null +++ b/bindings/go/vcencrypt/typed_null_test.go @@ -0,0 +1,155 @@ +package vcencrypt_test + +import ( + "reflect" + "testing" + "time" + + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// optional is a record with an optional field of every fixed-width type, +// as pointers, the Go side of Rust's Option. +type optional struct { + I8 *int8 + I16 *int16 + I32 *int32 + I64 *int64 + I128 *vcvalue.Int128 + U8 *uint8 + U16 *uint16 + U32 *uint32 + U64 *uint64 + U128 *vcvalue.Uint128 + F32 *float32 + F64 *float64 + Born *vcvalue.Date + Seen *time.Time + Price *vcvalue.Decimal + ID *vcvalue.UUID +} + +// Through the real guest: a nil fixed-width field seals to a leaf of the +// same length as a present one (vitaminc#493), so the ciphertext does not +// show which optional values are absent, and it decrypts back as nil. +func TestANilFixedWidthFieldSealsAsLongAsAPresentOne(t *testing.T) { + cipher := newCipher(t) + aad := []byte("typed-null") + var ( + i8 int8 = -1 + i16 int16 = -300 + i32 int32 = 36 + i64 int64 = 1 << 40 + i128 = vcvalue.Int128{1} + u8 uint8 = 1 + u16 uint16 = 300 + u32 uint32 = 70_000 + u64 uint64 = 1 << 40 + u128 = vcvalue.Uint128{1} + f32 float32 = 1.5 + f64 = 2.5 + born = vcvalue.Date{Year: 1815, Month: 12, Day: 10} + seen = time.Unix(1_700_000_000, 5).UTC() + price = vcvalue.Decimal{Scale: 2, Coefficient: [12]byte{150}} + id = vcvalue.UUID{0xAB} + ) + + lengths := func(v optional) map[string]int { + ct, err := cipher.Encrypt(t.Context(), v, aad) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + out := map[string]int{} + for key, leaf := range ct.(map[string]any) { + out[key] = len(leaf.(vcvalue.Sealed)) + } + got, err := cipher.Decrypt(t.Context(), ct, aad) + if err != nil { + t.Fatalf("Decrypt: %v", err) + } + if v.I32 == nil { + for _, f := range got.(vcvalue.Object) { + if f.Value != nil { + t.Fatalf("%s decrypted as %#v, want nil", f.Key, f.Value) + } + } + } + return out + } + present := lengths(optional{ + &i8, &i16, &i32, &i64, &i128, &u8, &u16, &u32, &u64, &u128, &f32, &f64, + &born, &seen, &price, &id, + }) + absent := lengths(optional{}) + if len(present) != 16 { + t.Fatalf("sealed %d fields", len(present)) + } + for key, n := range present { + if absent[key] != n { + t.Fatalf("%s: absent seals to %d bytes, present to %d", key, absent[key], n) + } + } +} + +// A nil fixed-width pointer inside vcvalue.Plain is a typed null in the +// frame the guest reads, and the guest writes it back as the plain null: +// a passthrough is never sealed, so the padding would hide nothing there, +// and it reads as nil in the ciphertext and after a decrypt. +func TestANilFixedWidthPointerInAPassthroughIsThePlainNull(t *testing.T) { + cipher := newCipher(t) + aad := []byte("plain-typed-null") + ct, err := cipher.Encrypt(t.Context(), map[string]any{ + "score": vcvalue.Plain{V: (*int32)(nil)}, + "email": "ada@example.com", + }, aad) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + if score := ct.(map[string]any)["score"]; score != (vcvalue.Plain{V: nil}) { + t.Fatalf("passthrough in the ciphertext: %#v", score) + } + got, err := cipher.Decrypt(t.Context(), ct, aad) + if err != nil { + t.Fatalf("Decrypt: %v", err) + } + want := vcvalue.Object{ + {Key: "email", Value: "ada@example.com"}, + {Key: "score", Value: vcvalue.Plain{V: nil}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v, want %#v", got, want) + } +} + +// A bool and a string keep the plain null: a bool is already as short as +// one, and text is variable width. +func TestANilBoolOrStringSealsThePlainNull(t *testing.T) { + cipher := newCipher(t) + aad := []byte("plain-null") + type rec struct { + On *bool + Name *string + } + ct, err := cipher.Encrypt(t.Context(), rec{}, aad) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + on := true + present, err := cipher.Encrypt(t.Context(), rec{On: &on}, aad) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + if got, want := len(ct.(map[string]any)["On"].(vcvalue.Sealed)), len(present.(map[string]any)["On"].(vcvalue.Sealed)); got != want { + t.Fatalf("bool: absent %d, present %d", got, want) + } + // A nil *string seals the plain null: as long as an empty string, and + // shorter than any other. + empty := "" + withName, err := cipher.Encrypt(t.Context(), rec{Name: &empty}, aad) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + if got, want := len(ct.(map[string]any)["Name"].(vcvalue.Sealed)), len(withName.(map[string]any)["Name"].(vcvalue.Sealed)); got != want { + t.Fatalf("string: absent %d, empty %d", got, want) + } +} diff --git a/bindings/go/vcencrypt/wasm/vitaminc_guest.wasm b/bindings/go/vcencrypt/wasm/vitaminc_guest.wasm index cacd0e62..6c2b63ce 100755 Binary files a/bindings/go/vcencrypt/wasm/vitaminc_guest.wasm and b/bindings/go/vcencrypt/wasm/vitaminc_guest.wasm differ diff --git a/bindings/go/vcffi/README.md b/bindings/go/vcffi/README.md index bb6750f4..4eb37a11 100644 --- a/bindings/go/vcffi/README.md +++ b/bindings/go/vcffi/README.md @@ -167,6 +167,64 @@ so that Rust and Node read it as a UUID. Matching explicit Encoder channels are available for custom `Encryptable` implementations. +### Absent values + +AES-GCM keeps the plaintext's length, so a one-byte null sealed in place of a +date would show which rows have no date. A nil pointer to a fixed-width type +(`*int32`, `*float64`, `*time.Time`, `*vcvalue.Date`, `*vcvalue.Decimal`, +`*vcvalue.UUID`, and so on, and a named int or float type such as +`type Age int32`) is therefore +written as a **typed null**, padded to the length of a value of that type, as +Rust writes an `Option`'s `None` (vitaminc#493): + +```go +type Person struct { + Name string + Born *vcvalue.Date // nil seals as long as a date +} +``` + +A nil with no fixed-width type to go by is the plain null: a nil interface, a +nil `*bool` (a bool is already as short as a null), a nil `*string` or +`*[]byte` (variable width), a nil pointer to an `Encryptable`, whose +encoding is its own, and a named type over a `vcvalue` or `time` type (`type +Birthday vcvalue.Date`), which reflection writes field by field, not as a +date. Where the Go type cannot say what is missing (a nil in a +`map[string]any`), write `vcvalue.TypedNull{Type: vcvalue.TypeDate}`, or call +`Encoder.TypedNull` from an `Encryptable`. + +An `Encryptable` whose `EncryptValue` always writes one value of one +fixed-width type can say so by implementing `FixedWidth`, the Go side of +Rust's `Encrypt::FIXED`. A nil pointer to it is then a typed null of that +type: + +```go +type Money struct{ cents int64 } + +func (m Money) EncryptValue(enc vcffi.Encoder) error { enc.Int64(m.cents); return nil } +func (Money) FixedType() vcvalue.FixedType { return vcvalue.TypeInt64 } + +type Order struct { + Refund *Money // nil seals as long as a Money +} +``` + +`FixedType` is called once per type, on a pointer to a zero value, so it must +answer from the type alone. A type that returns an invalid `FixedType` fails +the encode with `vcvalue.ErrInvalidFixedType` rather than write the plain +null in its place. + +A typed null decodes as `nil`, like every null. Out of the seal the padding +means nothing, and Rust's frame writer writes it back as the plain null, so +Go does too. A decrypted value therefore no longer says what type an absent +field was: hold optional fields in a typed struct (`Born *vcvalue.Date`) +before encrypting again. A value re-encrypted from `any` or `vcvalue.Object` +seals its nils as the plain null, and its absent values show by length. + +Reading a typed null takes a guest built from the vitaminc release that adds +it (vitaminc#493); an older one refuses the frame. A binding that ships its +own guest must update it together with this module. + The codec reads `testdata/value-conformance.json` from the repo root directly, as do Rust and Node. Invalid scalar widths, date ranges, nanoseconds, decimal flags and scales return `ErrMalformed`. Value construction errors retain the diff --git a/bindings/go/vcffi/decode.go b/bindings/go/vcffi/decode.go index 13c45b98..8eaf3e45 100644 --- a/bindings/go/vcffi/decode.go +++ b/bindings/go/vcffi/decode.go @@ -12,6 +12,8 @@ import ( // Unmarshal decodes value transport bytes into Go natives: // // - Null and Undefined → nil (Go has no undefined analog); +// - a typed null → nil: an absent value is absent whatever its type +// once it is out of the seal, so it re-encodes as the plain null; // - Bool → bool; // - Int32 → int32; // - Int64 → int64; @@ -169,6 +171,8 @@ func decodeTagged(r *reader, tag byte, depth int) (any, error) { return vcvalue.Plain{V: inner}, nil case tagEnvelope: return decodeEnvelope(r) + case tagTypedNull: + return decodeTypedNull(r) default: return nil, ErrMalformed } diff --git a/bindings/go/vcffi/encoder.go b/bindings/go/vcffi/encoder.go index 3dfcad2a..28f17c4d 100644 --- a/bindings/go/vcffi/encoder.go +++ b/bindings/go/vcffi/encoder.go @@ -8,6 +8,7 @@ import ( "reflect" "slices" "strings" + "sync" "time" "unicode/utf8" @@ -26,6 +27,22 @@ type Encryptable interface { EncryptValue(enc Encoder) error } +// FixedWidth is the Go side of Rust's `Encrypt::FIXED`: an Encryptable whose +// EncryptValue always writes one value of one fixed-width type (a money type +// that writes its cents with Int64, say). A nil pointer to such a type is +// written as a typed null of that type, so a sealed absence is as long as a +// sealed value. A nil pointer to an Encryptable that does not implement it +// is the plain null, since its encoding is its own. +// +// FixedType is called once per type, on a pointer to a zero value, so it +// must answer from the type alone. A type that returns an invalid FixedType +// fails the encode with vcvalue.ErrInvalidFixedType rather than write the +// plain null in its place. +type FixedWidth interface { + Encryptable + FixedType() vcvalue.FixedType +} + // encState is the buffer and first-error shared by an Encoder and every // sub-encoder it hands out. Errors accumulate: once one is recorded, further // channel calls are no-ops, so consumer code can chain writes without @@ -452,7 +469,10 @@ func Marshal(v any) ([]byte, error) { // - map[string]T → Map, keys SORTED for determinism; // - structs → Map by field name, honoring `vc:"name"` // tags (`vc:"-"` skips a field), in declaration order; -// - pointers → dereferenced (nil pointer → Null). +// - pointers → dereferenced; a nil pointer → a typed +// null if its element type is fixed width (see [fixedTypeOf]), Null +// otherwise; +// - vcvalue.TypedNull → a typed null of its type. // // Map keys are sorted because Go map iteration order is randomized; // sorting makes the encrypted structure deterministic. Struct fields keep @@ -496,43 +516,20 @@ func encodeAny(enc Encoder, v any) { // the record's structure. // - Encryptable hands control to the user type — except a typed nil // pointer, which still satisfies the assertion but would dereference nil -// in typical implementations; it encodes as Null like every other nil. +// in typical implementations; it encodes as Null like every other nil, +// or as a typed null when the type declares its width (FixedWidth). func interceptMarkers(enc Encoder, v any) bool { - switch x := v.(type) { - case vcvalue.Int128: - enc.Int128(x) - return true - case vcvalue.Uint128: - enc.UInt128(x) - return true - case vcvalue.Date: - enc.Date(x) - return true - case time.Time: - enc.Timestamp(x) - return true - case vcvalue.Timestamp: - enc.TimestampParts(x) - return true - case vcvalue.Decimal: - enc.Decimal(x) - return true - case vcvalue.UUID: - enc.UUID(x) - return true - case vcvalue.Tagged: - enc.Tagged(x) - return true - case vcvalue.Plain: - encodeAny(enc.Passthrough(), x.V) - return true - case vcvalue.Object: - encodeObject(enc, x) + if m, ok := markers[reflect.TypeOf(v)]; ok { + m.encode(enc, v) return true } if e, ok := v.(Encryptable); ok { if ev := reflect.ValueOf(e); ev.Kind() == reflect.Pointer && ev.IsNil() { - enc.Null() + if t, ok := fixedTypeOf(ev.Type()); ok { + enc.TypedNull(t) + } else { + enc.Null() + } return true } if err := e.EncryptValue(enc); err != nil { @@ -568,40 +565,13 @@ func encodeReflect(enc Encoder, rv reflect.Value) { return } + if k, ok := numericKinds[rv.Kind()]; ok { + k.encode(enc, rv) + return + } switch rv.Kind() { case reflect.Bool: enc.Bool(rv.Bool()) - case reflect.Int8: - enc.Int8(int8(rv.Int())) - case reflect.Int16: - enc.Int16(int16(rv.Int())) - case reflect.Int32: - enc.Int32(int32(rv.Int())) - case reflect.Int, reflect.Int64: - enc.Int64(rv.Int()) - case reflect.Uint8: - enc.UInt8(uint8(rv.Uint())) - case reflect.Uint16: - enc.UInt16(uint16(rv.Uint())) - case reflect.Uint32: - enc.UInt32(uint32(rv.Uint())) - case reflect.Uint, reflect.Uint64, reflect.Uintptr: - // Every unsigned word-or-wider kind keeps its unsigned identity as - // UInt64 (no fit-check: a bare uint always maps to UINT64 now). - enc.UInt64(rv.Uint()) - case reflect.Float32: - // rv.Float() widens to float64, and the hardware widening sets the - // quiet bit of a signaling NaN. Converting to the built-in float32 - // instead copies the bits (reflect special-cases float32 to float32, - // Go issue 36400), so a named float32 type keeps them too. Only a - // value that cannot be read as an interface falls back to widening. - if f, ok := interfaceOf(rv.Convert(float32Type)).(float32); ok { - enc.Float32(f) - } else { - enc.Float32(float32(rv.Float())) - } - case reflect.Float64: - enc.Float64(rv.Float()) case reflect.String: enc.String(rv.String()) case reflect.Slice: @@ -628,7 +598,16 @@ func encodeReflect(enc Encoder, rv reflect.Value) { encodeStruct(enc, rv) case reflect.Pointer, reflect.Interface: if rv.IsNil() { - enc.Null() + // A nil pointer still has its element type, so an absent + // fixed-width value is padded like Rust's Option. A nil + // interface has no type to go by. + if rv.Kind() != reflect.Pointer { + enc.Null() + } else if t, ok := fixedTypeOf(rv.Type()); ok { + enc.TypedNull(t) + } else { + enc.Null() + } return } // Charge the dereference against the shared depth budget: it writes @@ -704,10 +683,169 @@ func encodeStruct(enc Encoder, rv reflect.Value) { _ = m.End() } +// marker is how interceptMarkers writes a type it handles by exact type, +// and the fixed-width type of its typed null, if it has one (zero if not). +// One table for both, so a new marker type cannot be written one way and +// padded, or not, another. +type marker struct { + fixed vcvalue.FixedType + encode func(Encoder, any) +} + +// numeric is how encodeReflect writes an int, uint or float kind, and the +// fixed-width type of its typed null: the same one-table rule as marker. +type numeric struct { + fixed vcvalue.FixedType + encode func(Encoder, reflect.Value) +} + +var ( + encryptableType = reflect.TypeOf((*Encryptable)(nil)).Elem() + fixedWidthType = reflect.TypeOf((*FixedWidth)(nil)).Elem() + float32Type = reflect.TypeOf(float32(0)) + + // markers is filled in init: Plain and Object encode by calling back + // into encodeAny, which reads the table. + markers map[reflect.Type]marker + + numericKinds = map[reflect.Kind]numeric{ + reflect.Int8: {vcvalue.TypeInt8, func(e Encoder, rv reflect.Value) { e.Int8(int8(rv.Int())) }}, + reflect.Int16: {vcvalue.TypeInt16, func(e Encoder, rv reflect.Value) { e.Int16(int16(rv.Int())) }}, + reflect.Int32: {vcvalue.TypeInt32, func(e Encoder, rv reflect.Value) { e.Int32(int32(rv.Int())) }}, + reflect.Int: {vcvalue.TypeInt64, func(e Encoder, rv reflect.Value) { e.Int64(rv.Int()) }}, + reflect.Int64: {vcvalue.TypeInt64, func(e Encoder, rv reflect.Value) { e.Int64(rv.Int()) }}, + reflect.Uint8: {vcvalue.TypeUint8, func(e Encoder, rv reflect.Value) { e.UInt8(uint8(rv.Uint())) }}, + reflect.Uint16: {vcvalue.TypeUint16, func(e Encoder, rv reflect.Value) { + e.UInt16(uint16(rv.Uint())) + }}, + reflect.Uint32: {vcvalue.TypeUint32, func(e Encoder, rv reflect.Value) { + e.UInt32(uint32(rv.Uint())) + }}, + // Every unsigned word-or-wider kind keeps its unsigned identity as + // UInt64 (no fit-check: a bare uint always maps to UINT64). + reflect.Uint: {vcvalue.TypeUint64, func(e Encoder, rv reflect.Value) { e.UInt64(rv.Uint()) }}, + reflect.Uint64: {vcvalue.TypeUint64, func(e Encoder, rv reflect.Value) { e.UInt64(rv.Uint()) }}, + reflect.Uintptr: {vcvalue.TypeUint64, func(e Encoder, rv reflect.Value) { e.UInt64(rv.Uint()) }}, + reflect.Float32: {vcvalue.TypeFloat32, func(e Encoder, rv reflect.Value) { + // rv.Float() widens to float64, and the hardware widening sets + // the quiet bit of a signaling NaN. Converting to the built-in + // float32 instead copies the bits (reflect special-cases float32 + // to float32, Go issue 36400), so a named float32 type keeps + // them too. Only a value that cannot be read as an interface + // falls back to widening. + if f, ok := interfaceOf(rv.Convert(float32Type)).(float32); ok { + e.Float32(f) + } else { + e.Float32(float32(rv.Float())) + } + }}, + reflect.Float64: {vcvalue.TypeFloat64, func(e Encoder, rv reflect.Value) { e.Float64(rv.Float()) }}, + } + + // fixedTypes caches fixedTypeOf by pointer type: every nil pointer of a + // record type would otherwise pay its reflection again. + fixedTypes sync.Map +) + +func init() { + markers = map[reflect.Type]marker{ + reflect.TypeOf(vcvalue.Int128{}): {vcvalue.TypeInt128, func(e Encoder, v any) { + e.Int128(v.(vcvalue.Int128)) + }}, + reflect.TypeOf(vcvalue.Uint128{}): {vcvalue.TypeUint128, func(e Encoder, v any) { + e.UInt128(v.(vcvalue.Uint128)) + }}, + reflect.TypeOf(vcvalue.Date{}): {vcvalue.TypeDate, func(e Encoder, v any) { + e.Date(v.(vcvalue.Date)) + }}, + reflect.TypeOf(time.Time{}): {vcvalue.TypeTimestamp, func(e Encoder, v any) { + e.Timestamp(v.(time.Time)) + }}, + reflect.TypeOf(vcvalue.Timestamp{}): {vcvalue.TypeTimestamp, func(e Encoder, v any) { + e.TimestampParts(v.(vcvalue.Timestamp)) + }}, + reflect.TypeOf(vcvalue.Decimal{}): {vcvalue.TypeDecimal, func(e Encoder, v any) { + e.Decimal(v.(vcvalue.Decimal)) + }}, + reflect.TypeOf(vcvalue.UUID{}): {vcvalue.TypeUUID, func(e Encoder, v any) { + e.UUID(v.(vcvalue.UUID)) + }}, + // A tagged value's physical type is in the value, not the type. + reflect.TypeOf(vcvalue.Tagged{}): {0, func(e Encoder, v any) { e.Tagged(v.(vcvalue.Tagged)) }}, + reflect.TypeOf(vcvalue.TypedNull{}): {0, func(e Encoder, v any) { + e.TypedNull(v.(vcvalue.TypedNull).Type) + }}, + reflect.TypeOf(vcvalue.Plain{}): {0, func(e Encoder, v any) { + encodeAny(e.Passthrough(), v.(vcvalue.Plain).V) + }}, + reflect.TypeOf(vcvalue.Object{}): {0, func(e Encoder, v any) { + encodeObject(e, v.(vcvalue.Object)) + }}, + } +} + +// fixedAnswer is what fixedTypeOf caches: whether the type has a fixed-width +// type at all, and which. A type declared through FixedWidth is kept even +// when it is invalid, so that the encode fails instead of writing the plain +// null in its place. +type fixedAnswer struct { + ft vcvalue.FixedType + has bool +} + +// fixedTypeOf is the fixed-width type a pointer of type t points at, through +// any number of pointers, when encodeReflect writes every value of it with +// one fixed-width tag: the int, uint and float kinds (a named type over one +// included) and the fixed-width marker types, exactly (a named type over a +// marker is a struct to reflection, written field by field). A type that +// implements Encryptable chooses its own encoding, which may not be fixed +// width, so it has none unless it says so through FixedWidth. +func fixedTypeOf(t reflect.Type) (vcvalue.FixedType, bool) { + if cached, ok := fixedTypes.Load(t); ok { + a := cached.(fixedAnswer) + return a.ft, a.has + } + var a fixedAnswer + elem := t + // A pointer type that points at itself (type P *P) never reaches a + // non-pointer, so the walk is bounded by the same depth budget that + // bounds a value cycle. Past it the type has no fixed width. + for depth := 0; elem.Kind() == reflect.Pointer && !elem.Implements(encryptableType); depth++ { + if depth == maxDepth { + fixedTypes.Store(t, a) + return 0, false + } + elem = elem.Elem() + } + switch { + case elem.Implements(fixedWidthType): + a = fixedAnswer{zeroValueOf(elem).Interface().(FixedWidth).FixedType(), true} + case elem.Implements(encryptableType): + // Its encoding is its own. + default: + if m, ok := markers[elem]; ok { + a.ft = m.fixed + } else { + a.ft = numericKinds[elem.Kind()].fixed + } + a.has = a.ft != 0 + } + fixedTypes.Store(t, a) + return a.ft, a.has +} + +// zeroValueOf is a zero value of t that a method can be called on: for a +// pointer type, a pointer to a zero value rather than nil, so a method with +// a value receiver does not dereference nil. +func zeroValueOf(t reflect.Type) reflect.Value { + if t.Kind() == reflect.Pointer { + return reflect.New(t.Elem()) + } + return reflect.Zero(t) +} + // interfaceOf is rv.Interface(), or nil where reflection forbids it // (values reached through unexported fields). -var float32Type = reflect.TypeOf(float32(0)) - func interfaceOf(rv reflect.Value) any { if !rv.CanInterface() { return nil diff --git a/bindings/go/vcffi/fuzz_test.go b/bindings/go/vcffi/fuzz_test.go index ed085f3f..e3f71620 100644 --- a/bindings/go/vcffi/fuzz_test.go +++ b/bindings/go/vcffi/fuzz_test.go @@ -54,9 +54,10 @@ func addCorpusSeeds(f *testing.F, wrap func([]byte) []byte) { } } -// canonicalForm is what an accepted value encoding re-encodes as. Three +// canonicalForm is what an accepted value encoding re-encodes as. Four // things may legitimately change, and nothing else: an Undefined tag (0x01) -// comes back as Null (0x00), Go's documented projection; a decimal negative +// comes back as Null (0x00), Go's documented projection; a typed null comes +// back as Null too, as Rust's frame writer writes it; a decimal negative // zero comes back as zero with its scale kept (ADR 0001); and a timestamp in // the 0xF3 envelope (seconds, milliseconds or microseconds) comes back in // the shorthand nanosecond form, because Go reads it as a time.Time. The @@ -132,9 +133,25 @@ func canonicalValue(r *reader, out *[]byte) { case tagPassthrough: *out = append(*out, tag) canonicalValue(r, out) + case tagTypedNull: + // The header names the type, which fixes the padding; it all comes + // back as the plain null. + header, _ := r.byteTag() + width := fixedWidth(header) + if header == tagEnvelope { + b, _ := r.take(3) + if len(b) == 3 { + width = fixedWidth(b[2]) + if b[2] == tagBytes { + width = 16 + } + } + } + _, _ = r.take(width - 1) + *out = append(*out, tagNull) default: // Null, booleans and the fixed-width extended scalars. - b, _ := r.take(scalarWidth(tag)) + b, _ := r.take(fixedWidth(tag)) *out = append(append(*out, tag), b...) } } @@ -147,13 +164,13 @@ func shorthandTimestamp(r *reader, unit byte) ([]byte, bool) { var count *big.Int switch tag { case tagInt8, tagInt16, tagInt32, tagInt64, tagInt128: - b, _ := r.take(scalarWidth(tag) + map[byte]int{tagInt32: 4, tagInt64: 8}[tag]) + b, _ := r.take(fixedWidth(tag)) if len(b) == 0 { return nil, false } count = signedLittle(b) case tagUint8, tagUint16, tagUint32, tagUint64, tagUint128: - b, _ := r.take(scalarWidth(tag) + map[byte]int{tagUint32: 4, tagUint64: 8}[tag]) + b, _ := r.take(fixedWidth(tag)) if len(b) == 0 { return nil, false } diff --git a/bindings/go/vcffi/scalars.go b/bindings/go/vcffi/scalars.go index d142f981..449cda13 100644 --- a/bindings/go/vcffi/scalars.go +++ b/bindings/go/vcffi/scalars.go @@ -78,10 +78,14 @@ func (e Encoder) UUID(u vcvalue.UUID) { if e.st.err != nil { return } - e.push(tagEnvelope, logicalUUID, 0) + e.push(uuidEnvelope[:]...) e.Bytes(u[:]) } +// uuidEnvelope is the envelope header a UUID starts with, and the header of +// a UUID's typed null: the Uuid logical type, no parameter, over bytes. +var uuidEnvelope = [3]byte{tagEnvelope, logicalUUID, 0} + // Tagged records a physical scalar under a logical id this build does not // know, in the 0xF3 envelope: what Unmarshal produced for such a value, so // it goes back out unchanged. An id this build knows, or a value that is @@ -148,7 +152,7 @@ func (e Encoder) physical(v any) bool { } func decodeScalar(r *reader, tag byte) (any, error) { - b, err := r.take(scalarWidth(tag)) + b, err := r.take(fixedWidth(tag)) if err != nil { return nil, err } @@ -196,16 +200,18 @@ func decodeScalar(r *reader, tag byte) (any, error) { } } -// scalarWidth is the fixed payload width of an extended scalar tag, or zero -// for a tag decodeScalar does not handle. -func scalarWidth(tag byte) int { +// fixedWidth is the payload width of a fixed-width scalar tag, or zero for +// a tag whose payload is not fixed (or that has none). +func fixedWidth(tag byte) int { switch tag { case tagInt8, tagUint8: return 1 case tagInt16, tagUint16: return 2 - case tagDate: + case tagInt32, tagUint32, tagFloat32, tagDate: return 4 + case tagInt64, tagUint64, tagFloat64: + return 8 case tagTimestamp: return 12 case tagInt128, tagUint128, tagDecimal: @@ -213,3 +219,121 @@ func scalarWidth(tag byte) int { } return 0 } + +// numericTag reports whether tag is a plain int or float: what a typed null +// in the envelope may ride on under a timestamp unit or an unknown id. +func numericTag(tag byte, floats bool) bool { + switch tag { + case tagInt8, tagUint8, tagInt16, tagUint16, tagInt32, tagUint32, tagInt64, tagUint64, + tagInt128, tagUint128: + return true + case tagFloat32, tagFloat64: + return floats + } + return false +} + +// fixedTags is the tag each FixedType is written with, all but the UUID, +// whose header is uuidEnvelope over tagBytes. +var fixedTags = map[vcvalue.FixedType]byte{ + vcvalue.TypeInt8: tagInt8, + vcvalue.TypeInt16: tagInt16, + vcvalue.TypeInt32: tagInt32, + vcvalue.TypeInt64: tagInt64, + vcvalue.TypeInt128: tagInt128, + vcvalue.TypeUint8: tagUint8, + vcvalue.TypeUint16: tagUint16, + vcvalue.TypeUint32: tagUint32, + vcvalue.TypeUint64: tagUint64, + vcvalue.TypeUint128: tagUint128, + vcvalue.TypeFloat32: tagFloat32, + vcvalue.TypeFloat64: tagFloat64, + vcvalue.TypeDate: tagDate, + vcvalue.TypeTimestamp: tagTimestamp, + vcvalue.TypeDecimal: tagDecimal, +} + +// TypedNull records an absent value of a fixed-width type: a null padded to +// the length of a value of t, so the two seal to the same length. Encode +// writes one for a nil pointer to a fixed-width type; this channel is for a +// caller that knows the type some other way. An invalid t fails the +// encode. +func (e Encoder) TypedNull(t vcvalue.FixedType) { + if e.st.err != nil { + return + } + if !t.Valid() { + e.st.fail(vcvalue.ErrInvalidFixedType) + return + } + var header []byte + var payload int + if t == vcvalue.TypeUUID { + header, payload = append(uuidEnvelope[:], tagBytes), len(vcvalue.UUID{}) + } else { + tag := fixedTags[t] + header, payload = []byte{tag}, fixedWidth(tag) + } + e.push(tagTypedNull) + e.push(header...) + // One byte of the value's length went to tagTypedNull. + e.push(make([]byte, payload-1)...) + e.complete() +} + +// decodeTypedNull reads a typed null after its tag, as strictly as the Rust +// codec does, so every absence has one encoding: a header a value of some +// fixed-width type starts with, then exactly the zero padding to that +// value's length. It decodes as nil. A frame is never sealed, so the padding +// means nothing once the value is out, and Rust's frame writer writes the +// plain null for it too. +func decodeTypedNull(r *reader) (any, error) { + tag, err := r.byteTag() + if err != nil { + return nil, err + } + var payload int + switch { + case tag == tagEnvelope: + h, err := r.take(3) + if err != nil { + return nil, err + } + id, param, physical := h[0], h[1], h[2] + switch id { + case logicalTimestamp: + // The nanosecond unit has the shorthand Timestamp tag. + if param > unitMicros || !numericTag(physical, false) { + return nil, ErrMalformed + } + payload = fixedWidth(physical) + case logicalUUID: + if param != 0 || physical != tagBytes { + return nil, ErrMalformed + } + payload = 16 + case 0, logicalUndefined, logicalDate, logicalDecimal: + return nil, ErrMalformed + default: + // A logical id this build does not know, over an int or float. + if !numericTag(physical, true) { + return nil, ErrMalformed + } + payload = fixedWidth(physical) + } + case tag == tagDate || tag == tagTimestamp || tag == tagDecimal || numericTag(tag, true): + payload = fixedWidth(tag) + default: + return nil, ErrMalformed + } + padding, err := r.take(payload - 1) + if err != nil { + return nil, err + } + for _, b := range padding { + if b != 0 { + return nil, ErrMalformed + } + } + return nil, nil +} diff --git a/bindings/go/vcffi/transport.go b/bindings/go/vcffi/transport.go index 83f4d0ee..6ad1a9bb 100644 --- a/bindings/go/vcffi/transport.go +++ b/bindings/go/vcffi/transport.go @@ -35,6 +35,10 @@ const ( tagDate = 0x12 tagTimestamp = 0x13 tagDecimal = 0x14 + // tagTypedNull is an absent value of a fixed-width type, padded to that + // type's leaf length: the tag, the header a value of the type starts + // with, then zeros (vitaminc#493). + tagTypedNull = 0x15 tagArray = 0xF0 tagObject = 0xF1 // tagPassthrough wraps one value node that travels in the clear diff --git a/bindings/go/vcffi/typed_null_test.go b/bindings/go/vcffi/typed_null_test.go new file mode 100644 index 00000000..8f902e6b --- /dev/null +++ b/bindings/go/vcffi/typed_null_test.go @@ -0,0 +1,410 @@ +package vcffi + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// typedNullAnswers pins each FixedType's typed null: the cross-language +// bytes the Rust codec writes for the same type (vitaminc-aead's leaf +// tests), beside a value of the type, which must be as long. +var typedNullAnswers = []struct { + t vcvalue.FixedType + hex string + present any +}{ + {vcvalue.TypeInt8, "150c", int8(-1)}, + {vcvalue.TypeInt16, "150e00", int16(-1)}, + {vcvalue.TypeInt32, "1504000000", int32(-1)}, + {vcvalue.TypeInt64, "1505" + zeros(7), int64(-1)}, + {vcvalue.TypeInt128, "1510" + zeros(15), vcvalue.Int128{1}}, + {vcvalue.TypeUint8, "150d", uint8(1)}, + {vcvalue.TypeUint16, "150f00", uint16(1)}, + {vcvalue.TypeUint32, "1506000000", uint32(1)}, + {vcvalue.TypeUint64, "1507" + zeros(7), uint64(1)}, + {vcvalue.TypeUint128, "1511" + zeros(15), vcvalue.Uint128{1}}, + {vcvalue.TypeFloat32, "1508000000", float32(1.5)}, + {vcvalue.TypeFloat64, "1509" + zeros(7), 1.5}, + {vcvalue.TypeDate, "1512000000", vcvalue.Date{Year: 1999, Month: 12, Day: 31}}, + {vcvalue.TypeTimestamp, "1513" + zeros(11), time.Unix(1_700_000_000, 5).UTC()}, + {vcvalue.TypeDecimal, "1514" + zeros(15), vcvalue.Decimal{Scale: 2, Coefficient: [12]byte{150}}}, + {vcvalue.TypeUUID, "15f305000b" + zeros(15), vcvalue.UUID{0xAB}}, +} + +func zeros(n int) string { return hex.EncodeToString(make([]byte, n)) } + +func TestTypedNullKnownAnswers(t *testing.T) { + for _, row := range typedNullAnswers { + t.Run(fmt.Sprint(row.t), func(t *testing.T) { + got, err := Marshal(vcvalue.TypedNull{Type: row.t}) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(got) != row.hex { + t.Fatalf("got %x, want %s", got, row.hex) + } + present, err := Marshal(row.present) + if err != nil { + t.Fatal(err) + } + // A frame gives bytes a length prefix; a typed null of a UUID + // has none, because its sealed leaf has none. + want := len(present) + if row.t == vcvalue.TypeUUID { + want -= 4 + } + if len(got) != want { + t.Fatalf("typed null is %d bytes, a value %d", len(got), want) + } + // It reads back as nil, like every null, and writes back as one. + v, err := Unmarshal(got) + if err != nil || v != nil { + t.Fatalf("decoded %#v, %v", v, err) + } + }) + } + if len(typedNullAnswers) != int(vcvalue.TypeUUID) { + t.Fatalf("pinned %d of %d types", len(typedNullAnswers), vcvalue.TypeUUID) + } +} + +func TestAnInvalidFixedTypeFailsTheEncode(t *testing.T) { + for _, ft := range []vcvalue.FixedType{0, vcvalue.TypeUUID + 1, 255} { + if _, err := Marshal(vcvalue.TypedNull{Type: ft}); !errors.Is(err, vcvalue.ErrInvalidFixedType) { + t.Fatalf("%v: got %v", ft, err) + } + } +} + +type selfEncoded int32 + +func (s selfEncoded) EncryptValue(enc Encoder) error { + enc.String("custom") + return nil +} + +// An Encryptable with a pointer receiver: only *ptrEncoded implements it. +type ptrEncoded int32 + +func (p *ptrEncoded) EncryptValue(enc Encoder) error { + enc.String("custom") + return nil +} + +// A pointer type that points at itself never reaches a non-pointer. +type selfPointer *selfPointer + +// A nil pointer to a fixed-width type is a typed null, the Go side of +// Rust's Option; a nil with no fixed-width type to go by is the plain +// null. Each case is encoded twice, since the second encode reads the +// per-type cache. +func TestANilPointerIsATypedNullWhenItsTypeIsFixedWidth(t *testing.T) { + type age int32 + var missing *int32 + cases := []struct { + name string + v any + want string + }{ + {"*int8", (*int8)(nil), "150c"}, + {"*int16", (*int16)(nil), "150e00"}, + {"*int32", (*int32)(nil), "1504000000"}, + {"*int64", (*int64)(nil), "1505" + zeros(7)}, + {"*int", (*int)(nil), "1505" + zeros(7)}, + {"*uint8", (*uint8)(nil), "150d"}, + {"*uint16", (*uint16)(nil), "150f00"}, + {"*uint32", (*uint32)(nil), "1506000000"}, + {"*uint64", (*uint64)(nil), "1507" + zeros(7)}, + {"*uint", (*uint)(nil), "1507" + zeros(7)}, + {"*float32", (*float32)(nil), "1508000000"}, + {"*float64", (*float64)(nil), "1509" + zeros(7)}, + {"*time.Time", (*time.Time)(nil), "1513" + zeros(11)}, + {"*vcvalue.Date", (*vcvalue.Date)(nil), "1512000000"}, + {"*vcvalue.Timestamp", (*vcvalue.Timestamp)(nil), "1513" + zeros(11)}, + {"*vcvalue.Decimal", (*vcvalue.Decimal)(nil), "1514" + zeros(15)}, + {"*vcvalue.UUID", (*vcvalue.UUID)(nil), "15f305000b" + zeros(15)}, + {"*vcvalue.Int128", (*vcvalue.Int128)(nil), "1510" + zeros(15)}, + {"*vcvalue.Uint128", (*vcvalue.Uint128)(nil), "1511" + zeros(15)}, + {"named *age", (*age)(nil), "1504000000"}, + {"**int32", (**int32)(nil), "1504000000"}, + {"&(nil *int32)", &missing, "1504000000"}, + // No fixed width: the plain null. + {"*bool", (*bool)(nil), "00"}, + {"*string", (*string)(nil), "00"}, + {"*[]byte", (*[]byte)(nil), "00"}, + {"*[16]byte", (*[16]byte)(nil), "00"}, + {"*struct", (*struct{ A int32 })(nil), "00"}, + {"*Encryptable", (*selfEncoded)(nil), "00"}, + {"**Encryptable", (**selfEncoded)(nil), "00"}, + {"*pointer-receiver Encryptable", (*ptrEncoded)(nil), "00"}, + {"**pointer-receiver Encryptable", (**ptrEncoded)(nil), "00"}, + {"*any", (*any)(nil), "00"}, + {"self-pointing type", selfPointer(nil), "00"}, + {"nil", nil, "00"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for range 2 { + got, err := Marshal(c.v) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(got) != c.want { + t.Fatalf("got %x, want %s", got, c.want) + } + } + }) + } +} + +// An Encryptable that always writes one value of one fixed-width type: +// a FixedWidth, the Go side of Rust's Encrypt::FIXED. +type money struct{ cents int64 } + +func (m money) EncryptValue(enc Encoder) error { + enc.Int64(m.cents) + return nil +} + +func (money) FixedType() vcvalue.FixedType { return vcvalue.TypeInt64 } + +// The same, declared on the pointer. +type ptrMoney struct{ cents int64 } + +func (m *ptrMoney) EncryptValue(enc Encoder) error { + enc.Int64(m.cents) + return nil +} + +func (*ptrMoney) FixedType() vcvalue.FixedType { return vcvalue.TypeInt64 } + +// A FixedWidth that declares a type that is not one. +type badWidth struct{} + +func (badWidth) EncryptValue(enc Encoder) error { + enc.String("custom") + return nil +} + +func (badWidth) FixedType() vcvalue.FixedType { return 0 } + +func TestANilPointerToAFixedWidthEncryptableIsATypedNullOfItsType(t *testing.T) { + want := "1505" + zeros(7) + for _, c := range []struct { + name string + v any + }{ + {"*money", (*money)(nil)}, + {"**money", (**money)(nil)}, + {"*ptrMoney", (*ptrMoney)(nil)}, + {"**ptrMoney", (**ptrMoney)(nil)}, + } { + t.Run(c.name, func(t *testing.T) { + got, err := Marshal(c.v) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(got) != want { + t.Fatalf("got %x, want %s", got, want) + } + }) + } + // A present value is as long, and still written by EncryptValue. + present, err := Marshal(&money{cents: 150}) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(present) != "059600000000000000" { + t.Fatalf("present: got %x", present) + } + // Through a struct field, where a nil would otherwise show by length. + type order struct{ Refund *money } + absent, err := Marshal(order{}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasSuffix(absent, []byte{tagTypedNull, tagInt64, 0, 0, 0, 0, 0, 0, 0}) { + t.Fatalf("absent: %x", absent) + } + // A declared type that is not a fixed-width type fails the encode, and + // never writes the plain null in its place. A present value is not + // asked. + if _, err := Marshal((*badWidth)(nil)); !errors.Is(err, vcvalue.ErrInvalidFixedType) { + t.Fatalf("invalid declared type: got %v", err) + } + if _, err := Marshal(badWidth{}); err != nil { + t.Fatalf("present: %v", err) + } +} + +func TestAStructsNilFieldsAreTypedNulls(t *testing.T) { + type person struct { + Name string + Born *vcvalue.Date + Age *int32 + Tags []string + Any any + } + got, err := Marshal(person{Name: "Ada"}) + if err != nil { + t.Fatal(err) + } + v, err := Unmarshal(got) + if err != nil { + t.Fatal(err) + } + for _, f := range v.(vcvalue.Object) { + if (f.Key == "Born" || f.Key == "Age" || f.Key == "Any") && f.Value != nil { + t.Fatalf("%s decoded %#v", f.Key, f.Value) + } + } + // Born and Age carry their padding; Any has no type to go by, and a nil + // slice is an empty sequence, as it always was. + if !bytes.Contains(got, []byte{tagTypedNull, tagDate, 0, 0, 0}) || + !bytes.Contains(got, []byte{tagTypedNull, tagInt32, 0, 0, 0}) { + t.Fatalf("no typed nulls in %x", got) + } +} + +// The explicit channel, for an Encryptable that knows the type. +type maybeAge struct{ v *int32 } + +func (m maybeAge) EncryptValue(enc Encoder) error { + if m.v == nil { + enc.TypedNull(vcvalue.TypeInt32) + } else { + enc.Int32(*m.v) + } + return nil +} + +func TestTheTypedNullChannel(t *testing.T) { + got, err := Marshal(maybeAge{}) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(got) != "1504000000" { + t.Fatalf("got %x", got) + } + // It completes exactly one slot, like every terminal. + enc := func(e Encoder) error { + s := e.Seq() + s.Elem().TypedNull(vcvalue.TypeUint8) + s.Elem().Null() + return s.End() + } + got, err = Marshal(encryptableFunc(enc)) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(got) != "f002000000150d00" { + t.Fatalf("got %x", got) + } +} + +type encryptableFunc func(Encoder) error + +func (f encryptableFunc) EncryptValue(e Encoder) error { return f(e) } + +// Every absence has one encoding: the decoder refuses anything a writer +// does not write. The shared corpus pins the same refusals for every +// language; these are the Go-specific edges around them. +func TestATypedNullIsReadStrictly(t *testing.T) { + refused := []string{ + "15", "1504", "15040000", "1504000001", "150400000000", + "1500", "1502", "150a", "150b" + zeros(15), "1515040000", + "15f3", "15f303", "15f30301", "15f3030304" + zeros(3), "15f3030409" + zeros(7), + "15f3050104" + zeros(3), "15f3c8000a", "15f3c8000b" + zeros(15), + "f0010000001502", + } + for _, h := range refused { + raw, _ := hex.DecodeString(h) + if _, err := Unmarshal(raw); err != ErrMalformed { + t.Fatalf("%s: got %v, want ErrMalformed", h, err) + } + } + accepted := []string{ + "15f3030004000000", // seconds over an int32 + "15f3030210" + zeros(15), // microseconds over an int128 + "15f3c8ff09" + zeros(7), // an unknown id, any parameter, over a float64 + "f0010000001506000000", // inside an array + "f2150c", // as passthrough + "f101000000010000006115120000" + "00", // as an object value + } + for _, h := range accepted { + raw, _ := hex.DecodeString(h) + if _, err := Unmarshal(raw); err != nil { + t.Fatalf("%s: %v", h, err) + } + } +} + +// Every fixed-width marker and kind writes its values with the header its +// typed null carries, so the one table cannot pad a type as something it is +// not. A marker added without a sample here fails the test. +func TestEveryFixedTypeIsWrittenWithItsTypedNullsHeader(t *testing.T) { + samples := map[reflect.Type]any{ + reflect.TypeOf(vcvalue.Int128{}): vcvalue.Int128{1}, + reflect.TypeOf(vcvalue.Uint128{}): vcvalue.Uint128{1}, + reflect.TypeOf(vcvalue.Date{}): vcvalue.Date{Year: 2000, Month: 1, Day: 1}, + reflect.TypeOf(time.Time{}): time.Unix(7, 0), + reflect.TypeOf(vcvalue.Timestamp{}): vcvalue.Timestamp{Seconds: 7}, + reflect.TypeOf(vcvalue.Decimal{}): vcvalue.Decimal{Scale: 1}, + reflect.TypeOf(vcvalue.UUID{}): vcvalue.UUID{1}, + } + check := func(name string, ft vcvalue.FixedType, sample any) { + null, err := Marshal(vcvalue.TypedNull{Type: ft}) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + value, err := Marshal(sample) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + // The typed null's header (after its tag) begins the value's. + header := null[1:] + for len(header) > 0 && header[len(header)-1] == 0 { + header = header[:len(header)-1] + } + if !bytes.HasPrefix(value, header) { + t.Fatalf("%s: value %x does not start with its typed null's header %x", name, value, header) + } + } + // Every sample is of a fixed-width marker, so a marker that loses its + // fixed type is caught, not skipped. + for typ := range samples { + if markers[typ].fixed == 0 { + t.Fatalf("%v has a sample but no typed null", typ) + } + } + for typ, m := range markers { + if m.fixed == 0 { + continue + } + sample, ok := samples[typ] + if !ok { + t.Fatalf("no sample for the fixed-width marker %v", typ) + } + check(typ.String(), m.fixed, sample) + } + for kind, n := range numericKinds { + check(kind.String(), n.fixed, reflect.Zero(kindTypes[kind]).Interface()) + } +} + +var kindTypes = map[reflect.Kind]reflect.Type{ + reflect.Int8: reflect.TypeOf(int8(0)), reflect.Int16: reflect.TypeOf(int16(0)), + reflect.Int32: reflect.TypeOf(int32(0)), reflect.Int: reflect.TypeOf(0), + reflect.Int64: reflect.TypeOf(int64(0)), reflect.Uint8: reflect.TypeOf(uint8(0)), + reflect.Uint16: reflect.TypeOf(uint16(0)), reflect.Uint32: reflect.TypeOf(uint32(0)), + reflect.Uint: reflect.TypeOf(uint(0)), reflect.Uint64: reflect.TypeOf(uint64(0)), + reflect.Uintptr: reflect.TypeOf(uintptr(0)), reflect.Float32: reflect.TypeOf(float32(0)), + reflect.Float64: reflect.TypeOf(float64(0)), +} diff --git a/bindings/go/vcvalue/doc.go b/bindings/go/vcvalue/doc.go index 628dfb92..612b7f86 100644 --- a/bindings/go/vcvalue/doc.go +++ b/bindings/go/vcvalue/doc.go @@ -12,7 +12,9 @@ // - Sealed is one encrypted leaf (version || nonce || ciphertext || tag), // the only frozen byte format in the model. SealedEmptySeq and // SealedEmptyMap are its authenticated marker siblings for empty -// composites; an absent value is a sealed null leaf (ADR 0001). This +// composites; an absent value is a sealed null leaf (ADR 0001), padded +// to the length of a value when its type is fixed width (TypedNull, +// and a nil pointer to such a type; vitaminc#493). This // sealed family is vitaminc-encrypt's materialization of the model: another binding (a stack-encrypt SDK, // say) defines its own distinct sealed-leaf types and wires them to the // codec via vcffi's LeafSet — a stack-encrypt leaf is never a Sealed. diff --git a/bindings/go/vcvalue/null.go b/bindings/go/vcvalue/null.go new file mode 100644 index 00000000..a9ebfd29 --- /dev/null +++ b/bindings/go/vcvalue/null.go @@ -0,0 +1,48 @@ +package vcvalue + +// FixedType names a fixed-width type: one whose every value seals to a leaf +// of the same length. An absent value of such a type is written as a typed +// null, padded to that length, so a sealed absence cannot be told from a +// sealed value by its length, as a plain null (one byte) could be. +// +// bool is fixed width too, but its leaf is already as short as a null's, +// and text and bytes are variable width, so neither has a FixedType. A UUID's +// sixteen bytes are fixed by its type, so it has one. +type FixedType uint8 + +// The fixed-width types, each named for the Go type it is written from. +// TypeInt64 and TypeUint64 cover int and uint too, as the codec writes them. +const ( + TypeInt8 FixedType = iota + 1 + TypeInt16 + TypeInt32 + TypeInt64 + TypeInt128 + TypeUint8 + TypeUint16 + TypeUint32 + TypeUint64 + TypeUint128 + TypeFloat32 + TypeFloat64 + TypeDate + TypeTimestamp + TypeDecimal + TypeUUID +) + +// Valid reports whether t is one of the named fixed-width types. +func (t FixedType) Valid() bool { + return t >= TypeInt8 && t <= TypeUUID +} + +// TypedNull is an absent value of a fixed-width type, for a caller whose Go +// type cannot say what is missing: a nil in a map[string]any or an []any. +// A nil pointer to a fixed-width type (*int32, *time.Time, *Date, …) is +// written as a typed null without it. +// +// It only encodes. A typed null decodes as nil, like every null: once +// decrypted, an absent value is absent whatever its type. +type TypedNull struct { + Type FixedType +} diff --git a/bindings/go/vcvalue/scalars.go b/bindings/go/vcvalue/scalars.go index ec1033e7..7eebaef2 100644 --- a/bindings/go/vcvalue/scalars.go +++ b/bindings/go/vcvalue/scalars.go @@ -15,6 +15,7 @@ var ( ErrNonFiniteDecimal = errors.New("vcvalue: decimal must be finite") ErrInvalidUUID = errors.New("vcvalue: invalid UUID") ErrInvalidTagged = errors.New("vcvalue: a tagged value must hold a physical scalar under an unknown logical id") + ErrInvalidFixedType = errors.New("vcvalue: not a fixed-width type") ) // Int128 is a signed two's-complement integer in little-endian byte order. diff --git a/packages/aead/src/leaf.rs b/packages/aead/src/leaf.rs index 1d789848..c0319f87 100644 --- a/packages/aead/src/leaf.rs +++ b/packages/aead/src/leaf.rs @@ -973,16 +973,29 @@ mod tests { assert!(checked > 20, "only {checked} scalar vectors"); // A normalized vector reads, and writes back as its canonical bytes. + // A typed null is written back as the plain null, as a frame writer + // writes it, and its typed form back as itself. + let mut typed_nulls = 0; for v in corpus["normalized"].as_array().expect("normalized") { let name = v["name"].as_str().expect("name"); let bytes = hex(v["transport"].as_str().expect("transport")); let canonical = hex(v["reencode"].as_str().expect("reencode")); - let ((ty, value), used) = read(&bytes, Form::Frame).expect(name); + let (item, used) = read_item(&bytes, Form::Frame).expect(name); assert_eq!(used, bytes.len(), "{name}"); let mut out = Vec::new(); - write(ty, value, Form::Frame, &mut out).expect(name); + match item { + Item::Scalar(ty, value) => write(ty, value, Form::Frame, &mut out).expect(name), + Item::TypedNull(ty) => { + write(None, Physical::Null, Form::Frame, &mut out).expect(name); + let mut typed = Vec::new(); + write_typed_null(ty, &mut typed); + assert_eq!(typed, bytes, "{name}"); + typed_nulls += 1; + } + } assert_eq!(out, canonical, "{name}"); } + assert!(typed_nulls > 5, "only {typed_nulls} typed null vectors"); for v in corpus["malformed"].as_array().expect("malformed") { let name = v["name"].as_str().expect("name"); @@ -991,7 +1004,7 @@ mod tests { continue; } // Refused outright, or a valid scalar followed by trailing bytes. - match read(&bytes, Form::Frame) { + match read_item(&bytes, Form::Frame) { Err(_) => {} Ok((_, used)) => assert!(used < bytes.len(), "{name}"), } diff --git a/testdata/value-conformance.json b/testdata/value-conformance.json index 4248e625..97475e1f 100644 --- a/testdata/value-conformance.json +++ b/testdata/value-conformance.json @@ -439,6 +439,132 @@ "transport": "f3c8", "rule": "ADR 0001 section 3: the envelope is id, parameter and a physical scalar; a parameter and payload cut off are refused" } +, + { + "name": "typed-null-truncated", + "transport": "15", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-no-padding", + "transport": "1506", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-short-padding", + "transport": "15060000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-nonzero-padding", + "transport": "1506000100", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-trailing", + "transport": "150600000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-null", + "transport": "1500", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-undefined", + "transport": "1501", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-bool", + "transport": "1502", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-text", + "transport": "150a", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-plain-bytes", + "transport": "150b000000000000000000000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-nested", + "transport": "151506000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-of-array", + "transport": "15f000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-envelope-over-shorthand", + "transport": "15f3020004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-envelope-timestamp-nanos", + "transport": "15f3030310000000000000000000000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-envelope-id-zero", + "transport": "15f3000004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-envelope-undefined", + "transport": "15f3010004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-envelope-decimal", + "transport": "15f3040004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-uuid-over-int", + "transport": "15f3050004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-uuid-with-length", + "transport": "15f305000b10000000000000000000000000000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-uuid-with-parameter", + "transport": "15f305010b000000000000000000000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-timestamp-over-float", + "transport": "15f303010900000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-unknown-over-bytes", + "transport": "15f3c8000b000000000000000000000000000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-unknown-over-bool", + "transport": "15f3c80002", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-inside-envelope", + "transport": "f3c800150c", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + }, + { + "name": "typed-null-0xf4", + "transport": "15f4c80004000000", + "rule": "Typed null (vitaminc#493): every absence has one encoding, so a typed null is TYPED_NULL, the header a value of a fixed-width type starts with, then zero padding to that value's length, and nothing else" + } ], "normalized": [ { @@ -450,6 +576,88 @@ "reencode_value": "0.00", "rule": "ADR 0001 section 3: a decimal negative zero reads back as zero with its scale kept, and zero is always written without the sign" } +, + { + "name": "typed-null-uint32", + "kind": "null", + "value": null, + "transport": "1506000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-int8", + "kind": "null", + "value": null, + "transport": "150c", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-float64", + "kind": "null", + "value": null, + "transport": "150900000000000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-date", + "kind": "null", + "value": null, + "transport": "1512000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-timestamp", + "kind": "null", + "value": null, + "transport": "15130000000000000000000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-decimal", + "kind": "null", + "value": null, + "transport": "1514000000000000000000000000000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-timestamp-millis", + "kind": "null", + "value": null, + "transport": "15f303010500000000000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-uuid", + "kind": "null", + "value": null, + "transport": "15f305000b000000000000000000000000000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + }, + { + "name": "typed-null-unknown-over-float32", + "kind": "null", + "value": null, + "transport": "15f3c80908000000", + "reencode": "00", + "reencode_value": null, + "rule": "Typed null (vitaminc#493): an absent value of a fixed-width type, padded to that type's leaf length so its sealed length matches a present value's. A frame is never sealed, so it reads as null and is written back as the plain null" + } ], "ciphertext_malformed": [ {