feat(vcffi): write a nil fixed-width pointer as a typed null, and read typed nulls

bindings/go/vcencrypt/README.md

@@ -231,7 +231,8 @@ these find inputs where both are self-consistent but differ from each other.
 
 - **`FuzzDifferentialValue`** checks that both sides accept or reject the
   same value bytes. When both accept, it checks that Rust re-encodes the
-  input exactly and that Go's re-encoding matches, allowing only Go's
+  input exactly, apart from a typed null, which both sides write back as
+  the plain null, and that Go's re-encoding matches, allowing only Go's
   `undefined` to `null` projection.
 - **`FuzzDifferentialCipherText`** checks that both sides accept or reject
   the same ciphertext bytes. Fuzzed seals never authenticate, so there is no

bindings/go/vcencrypt/differential_fuzz_test.go

@@ -129,6 +129,21 @@ func corpusCiphertexts(f *testing.F) [][]byte {
 	return out
 }
 
+// intWidths is the payload width of each int tag, which a timestamp in the
+// envelope rides on.
+var intWidths = map[byte]int{0x0C: 1, 0x0D: 1, 0x0E: 2, 0x0F: 2, 0x04: 4, 0x06: 4, 0x05: 8, 0x07: 8, 0x10: 16, 0x11: 16}
+
+// typedNullWidths is the payload width of each header a typed null may
+// carry: an int, a float, a date, a timestamp, a decimal, and bytes, which
+// only a UUID's typed null carries, sixteen wide.
+var typedNullWidths = func() map[byte]int {
+	m := map[byte]int{0x08: 4, 0x12: 4, 0x09: 8, 0x13: 12, 0x14: 16, 0x0B: 16}
+	for tag, width := range intWidths {
+		m[tag] = width
+	}
+	return m
+}()
+
 // requireGoMatchesRust fails unless goOut is rustOut with exactly Go's
 // documented projections applied: an Undefined value-node tag (0x01) turned
 // into Null (0x00), and a timestamp in the 0xF3 envelope (seconds,
@@ -144,16 +159,18 @@ func requireGoMatchesRust(t *testing.T, data, rustOut, goOut []byte) {
 }
 
 // rustCanonical is what Rust re-encodes an accepted value as: the input,
-// with a decimal negative zero read back as zero (ADR 0001). Rust keeps
-// Undefined and envelope timestamps as they are.
+// with a decimal negative zero read back as zero (ADR 0001) and a typed null
+// written as the plain null, since a frame is never sealed (vitaminc#493).
+// Rust keeps Undefined and envelope timestamps as they are.
 func rustCanonical(data []byte) []byte {
 	return rewrite(data, false)
 }
 
 // rewrite walks an accepted value tree following the transport format
 // independently of either decoder, so a walk that falls out of step with it
 // produces the wrong bytes and fails the comparison rather than passing. It
-// always normalizes a decimal negative zero; with goProjection it also
+// always normalizes a decimal negative zero and writes a typed null as the
+// plain null; with goProjection it also
 // applies Go's two projections. vcffi's fuzz tests have the same walker, but
 // this is a separate Go module and cannot import test code.
 func rewrite(data []byte, goProjection bool) []byte {
@@ -226,6 +243,18 @@ func rewrite(data []byte, goProjection bool) []byte {
 		case 0xF2: // Passthrough
 			out = append(out, tag)
 			walk()
+		case 0x15: // a typed null: both sides write it back as the plain null
+			header := take(1)
+			if len(header) == 1 && header[0] == 0xF3 {
+				// id and parameter, then the physical tag the width follows.
+				if envelope := take(3); len(envelope) == 3 {
+					header = envelope[2:]
+				}
+			}
+			if len(header) == 1 {
+				take(typedNullWidths[header[0]] - 1)
+			}
+			out = append(out, 0x00)
 		case 0xF3: // the envelope: id, parameter, then a physical scalar
 			header := take(2)
 			if goProjection && len(header) == 2 && header[0] == 3 && header[1] <= 2 {
@@ -253,8 +282,7 @@ func shorthandTimestamp(data []byte, pos *int, unit byte) ([]byte, bool) {
 		return nil, false
 	}
 	tag := data[*pos]
-	widths := map[byte]int{0x0C: 1, 0x0D: 1, 0x0E: 2, 0x0F: 2, 0x04: 4, 0x06: 4, 0x05: 8, 0x07: 8, 0x10: 16, 0x11: 16}
-	width, ok := widths[tag]
+	width, ok := intWidths[tag]
 	if !ok || *pos+1+width > len(data) {
 		return nil, false
 	}

bindings/go/vcencrypt/typed_null_test.go

@@ -0,0 +1,155 @@
+package vcencrypt_test
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cipherstash/vitaminc/bindings/go/vcvalue"
+)
+
+// optional is a record with an optional field of every fixed-width type,
+// as pointers, the Go side of Rust's Option<T>.
+type optional struct {
+	I8    *int8
+	I16   *int16
+	I32   *int32
+	I64   *int64
+	I128  *vcvalue.Int128
+	U8    *uint8
+	U16   *uint16
+	U32   *uint32
+	U64   *uint64
+	U128  *vcvalue.Uint128
+	F32   *float32
+	F64   *float64
+	Born  *vcvalue.Date
+	Seen  *time.Time
+	Price *vcvalue.Decimal
+	ID    *vcvalue.UUID
+}
+
+// Through the real guest: a nil fixed-width field seals to a leaf of the
+// same length as a present one (vitaminc#493), so the ciphertext does not
+// show which optional values are absent, and it decrypts back as nil.
+func TestANilFixedWidthFieldSealsAsLongAsAPresentOne(t *testing.T) {
+	cipher := newCipher(t)
+	aad := []byte("typed-null")
+	var (
+		i8    int8    = -1
+		i16   int16   = -300
+		i32   int32   = 36
+		i64   int64   = 1 << 40
+		i128          = vcvalue.Int128{1}
+		u8    uint8   = 1
+		u16   uint16  = 300
+		u32   uint32  = 70_000
+		u64   uint64  = 1 << 40
+		u128          = vcvalue.Uint128{1}
+		f32   float32 = 1.5
+		f64           = 2.5
+		born          = vcvalue.Date{Year: 1815, Month: 12, Day: 10}
+		seen          = time.Unix(1_700_000_000, 5).UTC()
+		price         = vcvalue.Decimal{Scale: 2, Coefficient: [12]byte{150}}
+		id            = vcvalue.UUID{0xAB}
+	)
+
+	lengths := func(v optional) map[string]int {
+		ct, err := cipher.Encrypt(t.Context(), v, aad)
+		if err != nil {
+			t.Fatalf("Encrypt: %v", err)
+		}
+		out := map[string]int{}
+		for key, leaf := range ct.(map[string]any) {
+			out[key] = len(leaf.(vcvalue.Sealed))
+		}
+		got, err := cipher.Decrypt(t.Context(), ct, aad)
+		if err != nil {
+			t.Fatalf("Decrypt: %v", err)
+		}
+		if v.I32 == nil {
+			for _, f := range got.(vcvalue.Object) {
+				if f.Value != nil {
+					t.Fatalf("%s decrypted as %#v, want nil", f.Key, f.Value)
+				}
+			}
+		}
+		return out
+	}
+	present := lengths(optional{
+		&i8, &i16, &i32, &i64, &i128, &u8, &u16, &u32, &u64, &u128, &f32, &f64,
+		&born, &seen, &price, &id,
+	})
+	absent := lengths(optional{})
+	if len(present) != 16 {
+		t.Fatalf("sealed %d fields", len(present))
+	}
+	for key, n := range present {
+		if absent[key] != n {
+			t.Fatalf("%s: absent seals to %d bytes, present to %d", key, absent[key], n)
+		}
+	}
+}
+
+// A nil fixed-width pointer inside vcvalue.Plain is a typed null in the
+// frame the guest reads, and the guest writes it back as the plain null:
+// a passthrough is never sealed, so the padding would hide nothing there,
+// and it reads as nil in the ciphertext and after a decrypt.
+func TestANilFixedWidthPointerInAPassthroughIsThePlainNull(t *testing.T) {
+	cipher := newCipher(t)
+	aad := []byte("plain-typed-null")
+	ct, err := cipher.Encrypt(t.Context(), map[string]any{
+		"score": vcvalue.Plain{V: (*int32)(nil)},
+		"email": "ada@example.com",
+	}, aad)
+	if err != nil {
+		t.Fatalf("Encrypt: %v", err)
+	}
+	if score := ct.(map[string]any)["score"]; score != (vcvalue.Plain{V: nil}) {
+		t.Fatalf("passthrough in the ciphertext: %#v", score)
+	}
+	got, err := cipher.Decrypt(t.Context(), ct, aad)
+	if err != nil {
+		t.Fatalf("Decrypt: %v", err)
+	}
+	want := vcvalue.Object{
+		{Key: "email", Value: "ada@example.com"},
+		{Key: "score", Value: vcvalue.Plain{V: nil}},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %#v, want %#v", got, want)
+	}
+}
+
+// A bool and a string keep the plain null: a bool is already as short as
+// one, and text is variable width.
+func TestANilBoolOrStringSealsThePlainNull(t *testing.T) {
+	cipher := newCipher(t)
+	aad := []byte("plain-null")
+	type rec struct {
+		On   *bool
+		Name *string
+	}
+	ct, err := cipher.Encrypt(t.Context(), rec{}, aad)
+	if err != nil {
+		t.Fatalf("Encrypt: %v", err)
+	}
+	on := true
+	present, err := cipher.Encrypt(t.Context(), rec{On: &on}, aad)
+	if err != nil {
+		t.Fatalf("Encrypt: %v", err)
+	}
+	if got, want := len(ct.(map[string]any)["On"].(vcvalue.Sealed)), len(present.(map[string]any)["On"].(vcvalue.Sealed)); got != want {
+		t.Fatalf("bool: absent %d, present %d", got, want)
+	}
+	// A nil *string seals the plain null: as long as an empty string, and
+	// shorter than any other.
+	empty := ""
+	withName, err := cipher.Encrypt(t.Context(), rec{Name: &empty}, aad)
+	if err != nil {
+		t.Fatalf("Encrypt: %v", err)
+	}
+	if got, want := len(ct.(map[string]any)["Name"].(vcvalue.Sealed)), len(withName.(map[string]any)["Name"].(vcvalue.Sealed)); got != want {
+		t.Fatalf("string: absent %d, empty %d", got, want)
+	}
+}

bindings/go/vcffi/README.md

@@ -167,6 +167,64 @@ so that Rust and Node read it as a UUID.
 Matching explicit Encoder channels are available for custom `Encryptable`
 implementations.
 
+### Absent values
+
+AES-GCM keeps the plaintext's length, so a one-byte null sealed in place of a
+date would show which rows have no date. A nil pointer to a fixed-width type
+(`*int32`, `*float64`, `*time.Time`, `*vcvalue.Date`, `*vcvalue.Decimal`,
+`*vcvalue.UUID`, and so on, and a named int or float type such as
+`type Age int32`) is therefore
+written as a **typed null**, padded to the length of a value of that type, as
+Rust writes an `Option<T>`'s `None` (vitaminc#493):
+
+```go
+type Person struct {
+	Name string
+	Born *vcvalue.Date // nil seals as long as a date
+}
+```
+
+A nil with no fixed-width type to go by is the plain null: a nil interface, a
+nil `*bool` (a bool is already as short as a null), a nil `*string` or
+`*[]byte` (variable width), a nil pointer to an `Encryptable`, whose
+encoding is its own, and a named type over a `vcvalue` or `time` type (`type
+Birthday vcvalue.Date`), which reflection writes field by field, not as a
+date. Where the Go type cannot say what is missing (a nil in a
+`map[string]any`), write `vcvalue.TypedNull{Type: vcvalue.TypeDate}`, or call
+`Encoder.TypedNull` from an `Encryptable`.
+
+An `Encryptable` whose `EncryptValue` always writes one value of one
+fixed-width type can say so by implementing `FixedWidth`, the Go side of
+Rust's `Encrypt::FIXED`. A nil pointer to it is then a typed null of that
+type:
+
+```go
+type Money struct{ cents int64 }
+
+func (m Money) EncryptValue(enc vcffi.Encoder) error { enc.Int64(m.cents); return nil }
+func (Money) FixedType() vcvalue.FixedType            { return vcvalue.TypeInt64 }
+
+type Order struct {
+	Refund *Money // nil seals as long as a Money
+}
+```
+
+`FixedType` is called once per type, on a pointer to a zero value, so it must
+answer from the type alone. A type that returns an invalid `FixedType` fails
+the encode with `vcvalue.ErrInvalidFixedType` rather than write the plain
+null in its place.
+
+A typed null decodes as `nil`, like every null. Out of the seal the padding
+means nothing, and Rust's frame writer writes it back as the plain null, so
+Go does too. A decrypted value therefore no longer says what type an absent
+field was: hold optional fields in a typed struct (`Born *vcvalue.Date`)
+before encrypting again. A value re-encrypted from `any` or `vcvalue.Object`
+seals its nils as the plain null, and its absent values show by length.
+
+Reading a typed null takes a guest built from the vitaminc release that adds
+it (vitaminc#493); an older one refuses the frame. A binding that ships its
+own guest must update it together with this module.
+
 The codec reads `testdata/value-conformance.json` from the repo root directly,
 as do Rust and Node. Invalid scalar widths, date ranges, nanoseconds, decimal
 flags and scales return `ErrMalformed`. Value construction errors retain the

bindings/go/vcffi/decode.go

@@ -12,6 +12,8 @@ import (
 // Unmarshal decodes value transport bytes into Go natives:
 //
 //   - Null and Undefined → nil (Go has no undefined analog);
+//   - a typed null       → nil: an absent value is absent whatever its type
+//     once it is out of the seal, so it re-encodes as the plain null;
 //   - Bool               → bool;
 //   - Int32              → int32;
 //   - Int64              → int64;
@@ -169,6 +171,8 @@ func decodeTagged(r *reader, tag byte, depth int) (any, error) {
 		return vcvalue.Plain{V: inner}, nil
 	case tagEnvelope:
 		return decodeEnvelope(r)
+	case tagTypedNull:
+		return decodeTypedNull(r)
 	default:
 		return nil, ErrMalformed
 	}