From 9f92081aed5d9043c18ca3ebab9b966b40e27f95 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:18:16 -0400 Subject: [PATCH 1/8] feat: add firehose field annotation proto extensions (transactions, nondeterministic) --- pb/firehose/options.pb.go | 116 ++++++++++++++++++++++++++++++++++++++ pb/firehose/options.proto | 17 ++++++ 2 files changed, 133 insertions(+) create mode 100644 pb/firehose/options.pb.go create mode 100644 pb/firehose/options.proto diff --git a/pb/firehose/options.pb.go b/pb/firehose/options.pb.go new file mode 100644 index 0000000..9adfed7 --- /dev/null +++ b/pb/firehose/options.pb.go @@ -0,0 +1,116 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc v6.32.1 +// source: pb/firehose/options.proto + +package pbfirehose + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var file_pb_firehose_options_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*descriptorpb.FieldOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 50000, + Name: "firehose.transactions", + Tag: "varint,50000,opt,name=transactions", + Filename: "pb/firehose/options.proto", + }, + { + ExtendedType: (*descriptorpb.FieldOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 50001, + Name: "firehose.nondeterministic", + Tag: "varint,50001,opt,name=nondeterministic", + Filename: "pb/firehose/options.proto", + }, +} + +// Extension fields to descriptorpb.FieldOptions. +var ( + // transactions marks the repeated field that holds the block's transaction list. + // Exactly one field per Block message should carry this annotation. + // + // optional bool transactions = 50000; + E_Transactions = &file_pb_firehose_options_proto_extTypes[0] + // nondeterministic marks fields whose values may differ between nodes + // (e.g. gas used, fees) and should be cleared for deterministic comparisons. + // + // optional bool nondeterministic = 50001; + E_Nondeterministic = &file_pb_firehose_options_proto_extTypes[1] +) + +var File_pb_firehose_options_proto protoreflect.FileDescriptor + +var file_pb_firehose_options_proto_rawDesc = string([]byte{ + 0x0a, 0x19, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x72, 0x65, 0x68, 0x6f, 0x73, 0x65, 0x2f, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x66, 0x69, 0x72, + 0x65, 0x68, 0x6f, 0x73, 0x65, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3a, 0x43, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xd0, 0x86, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3a, 0x4b, 0x0a, 0x10, + 0x6e, 0x6f, 0x6e, 0x64, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, + 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0xd1, 0x86, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x6e, 0x6f, 0x6e, 0x64, 0x65, 0x74, 0x65, + 0x72, 0x6d, 0x69, 0x6e, 0x69, 0x73, 0x74, 0x69, 0x63, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, + 0x67, 0x66, 0x61, 0x73, 0x74, 0x2f, 0x66, 0x69, 0x72, 0x65, 0x68, 0x6f, 0x73, 0x65, 0x2d, 0x63, + 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x72, 0x65, 0x68, 0x6f, 0x73, 0x65, 0x3b, + 0x70, 0x62, 0x66, 0x69, 0x72, 0x65, 0x68, 0x6f, 0x73, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +}) + +var file_pb_firehose_options_proto_goTypes = []any{ + (*descriptorpb.FieldOptions)(nil), // 0: google.protobuf.FieldOptions +} +var file_pb_firehose_options_proto_depIdxs = []int32{ + 0, // 0: firehose.transactions:extendee -> google.protobuf.FieldOptions + 0, // 1: firehose.nondeterministic:extendee -> google.protobuf.FieldOptions + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 0, // [0:2] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_pb_firehose_options_proto_init() } +func file_pb_firehose_options_proto_init() { + if File_pb_firehose_options_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_pb_firehose_options_proto_rawDesc), len(file_pb_firehose_options_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 2, + NumServices: 0, + }, + GoTypes: file_pb_firehose_options_proto_goTypes, + DependencyIndexes: file_pb_firehose_options_proto_depIdxs, + ExtensionInfos: file_pb_firehose_options_proto_extTypes, + }.Build() + File_pb_firehose_options_proto = out.File + file_pb_firehose_options_proto_goTypes = nil + file_pb_firehose_options_proto_depIdxs = nil +} diff --git a/pb/firehose/options.proto b/pb/firehose/options.proto new file mode 100644 index 0000000..753172b --- /dev/null +++ b/pb/firehose/options.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package firehose; + +import "google/protobuf/descriptor.proto"; + +option go_package = "github.com/streamingfast/firehose-core/pb/firehose;pbfirehose"; + +extend google.protobuf.FieldOptions { + // transactions marks the repeated field that holds the block's transaction list. + // Exactly one field per Block message should carry this annotation. + bool transactions = 50000; + + // nondeterministic marks fields whose values may differ between nodes + // (e.g. gas used, fees) and should be cleared for deterministic comparisons. + bool nondeterministic = 50001; +} From 295195f403d19021c2a5666526f2905043b49a26 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:24:18 -0400 Subject: [PATCH 2/8] docs: add regeneration instructions to firehose options.proto --- pb/firehose/options.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pb/firehose/options.proto b/pb/firehose/options.proto index 753172b..971a053 100644 --- a/pb/firehose/options.proto +++ b/pb/firehose/options.proto @@ -1,5 +1,9 @@ syntax = "proto3"; +// Regenerate options.pb.go from the repo root: +// PROTOBUF_INCLUDE=$(go list -m -json google.golang.org/protobuf | grep '"Dir"' | sed 's/.*"Dir": "\(.*\)".*/\1/') +// protoc --go_out=. --go_opt=paths=source_relative -I . -I ${PROTOBUF_INCLUDE}/src pb/firehose/options.proto + package firehose; import "google/protobuf/descriptor.proto"; From 815b71262a22a18e4a50ce0464f37f0e2b826e97 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:27:46 -0400 Subject: [PATCH 3/8] feat: add protox dependency with local replace for instance walker Add replace directive pointing github.com/streamingfast/protox to the local branch at /Users/maoueh/work/sf/protox which contains the new WalkMessageInstanceFields function needed for tagging chain proto fields. --- go.mod | 3 ++- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb2fc30..196d4ef 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/testcontainers/testcontainers-go v0.40.0 go.opentelemetry.io/otel v1.43.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.27.1 + go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 @@ -311,5 +311,6 @@ replace ( cloud.google.com/go => github.com/streamingfast/google-cloud-go v0.0.0-20241202194114-f77ff78d4f66 github.com/ShinyTrinkets/overseer => github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef github.com/jhump/protoreflect => github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d + github.com/streamingfast/protox => /Users/maoueh/work/sf/protox github.com/tetratelabs/wazero => github.com/streamingfast/wazero v0.0.0-20241202185309-91287c3640ed ) diff --git a/go.sum b/go.sum index 8e7c36c..7b9e54b 100644 --- a/go.sum +++ b/go.sum @@ -2521,8 +2521,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= From c7caabd84bd4abfbe702028c9a5dac4fbe9e3eae Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:35:42 -0400 Subject: [PATCH 4/8] feat: add fireproto package for walking and clearing non-deterministic proto fields Adds the fireproto package with WalkNonDeterministicFields, ClearNonDeterministicFields, and FindTransactionsField utilities that use the firehose field option extensions to locate and manipulate annotated proto fields without chain-specific knowledge. --- fireproto/doc.go | 37 +++++++++ fireproto/log_test.go | 9 ++ fireproto/walker.go | 45 ++++++++++ fireproto/walker_test.go | 173 +++++++++++++++++++++++++++++++++++++++ go.mod | 1 + 5 files changed, 265 insertions(+) create mode 100644 fireproto/doc.go create mode 100644 fireproto/log_test.go create mode 100644 fireproto/walker.go create mode 100644 fireproto/walker_test.go diff --git a/fireproto/doc.go b/fireproto/doc.go new file mode 100644 index 0000000..a4800f4 --- /dev/null +++ b/fireproto/doc.go @@ -0,0 +1,37 @@ +// Package fireproto provides utilities for working with Firehose-annotated +// protobuf messages using custom field extensions defined in the firehose proto +// package (pb/firehose/options.proto). +// +// # Field Annotations +// +// The firehose proto package defines two FieldOptions extensions that can be +// applied to fields in any Block protobuf message: +// +// - (firehose.transactions) — marks the repeated field that holds the block's +// transaction list, enabling generic tooling to locate transactions without +// chain-specific knowledge. +// +// - (firehose.nondeterministic) — marks fields whose values may differ between +// nodes (e.g. gas estimates, fee calculations, timing metadata). These fields +// are candidates for clearing when performing deterministic block comparisons +// or diffing block content across nodes. +// +// # Usage +// +// Walk all non-deterministic fields in a block: +// +// fireproto.WalkNonDeterministicFields(block, func(msg protoreflect.Message, field protoreflect.FieldDescriptor) { +// fmt.Printf("non-deterministic: %s.%s\n", msg.Descriptor().Name(), field.Name()) +// }) +// +// Clear all non-deterministic fields before comparison: +// +// fireproto.ClearNonDeterministicFields(block) +// +// Find the transactions field on a block: +// +// fd := fireproto.FindTransactionsField(block) +// if fd != nil { +// txs := block.ProtoReflect().Get(fd).List() +// } +package fireproto diff --git a/fireproto/log_test.go b/fireproto/log_test.go new file mode 100644 index 0000000..99453bf --- /dev/null +++ b/fireproto/log_test.go @@ -0,0 +1,9 @@ +package fireproto_test + +import "github.com/streamingfast/logging" + +var zlogTest, tracerTest = logging.PackageLogger("fireproto_test", "github.com/streamingfast/firehose-core/fireproto/test") + +func init() { + logging.InstantiateLoggers() +} diff --git a/fireproto/walker.go b/fireproto/walker.go new file mode 100644 index 0000000..52aa6fa --- /dev/null +++ b/fireproto/walker.go @@ -0,0 +1,45 @@ +package fireproto + +import ( + pbfirehose "github.com/streamingfast/firehose-core/pb/firehose" + "github.com/streamingfast/protox" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// WalkNonDeterministicFields walks the proto.Message tree rooted at root and calls fn +// for each field tagged with (firehose.nondeterministic) = true. +// fn receives the containing message instance and the field descriptor. +func WalkNonDeterministicFields(root proto.Message, fn func(msg protoreflect.Message, field protoreflect.FieldDescriptor)) { + for msg, field := range protox.WalkMessageInstanceFields(root.ProtoReflect(), nil) { + if isNonDeterministic(field) { + fn(msg, field) + } + } +} + +// ClearNonDeterministicFields clears all fields tagged with (firehose.nondeterministic) = true +// in the message tree rooted at root. Fields are set to their zero value. +func ClearNonDeterministicFields(root proto.Message) { + WalkNonDeterministicFields(root, func(msg protoreflect.Message, field protoreflect.FieldDescriptor) { + msg.Clear(field) + }) +} + +// FindTransactionsField returns the first field descriptor in root's message type +// tagged with (firehose.transactions) = true, or nil if none is tagged. +func FindTransactionsField(root proto.Message) protoreflect.FieldDescriptor { + fields := root.ProtoReflect().Descriptor().Fields() + for i := range fields.Len() { + field := fields.Get(i) + if v, ok := protox.GetFieldExtensionValue[bool](field, pbfirehose.E_Transactions, false); ok && v { + return field + } + } + return nil +} + +func isNonDeterministic(field protoreflect.FieldDescriptor) bool { + v, ok := protox.GetFieldExtensionValue[bool](field, pbfirehose.E_Nondeterministic, false) + return ok && v +} diff --git a/fireproto/walker_test.go b/fireproto/walker_test.go new file mode 100644 index 0000000..8ad28ed --- /dev/null +++ b/fireproto/walker_test.go @@ -0,0 +1,173 @@ +package fireproto_test + +import ( + "testing" + + "github.com/streamingfast/firehose-core/fireproto" + pbfirehose "github.com/streamingfast/firehose-core/pb/firehose" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/dynamicpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// buildAnnotatedDescriptors builds dynamic proto descriptors: +// +// Block { repeated Tx transactions = 1 [(firehose.transactions)=true]; bytes gas = 2 [(firehose.nondeterministic)=true]; } +// Tx { string hash = 1; bytes fee = 2 [(firehose.nondeterministic)=true]; } +func buildAnnotatedDescriptors(t *testing.T) (blockType protoreflect.MessageType, txType protoreflect.MessageType) { + t.Helper() + + txFDP := &descriptorpb.FileDescriptorProto{ + Name: proto.String("fireproto_test_tx.proto"), + Syntax: proto.String("proto3"), + Package: proto.String("fireproto.test"), + Options: &descriptorpb.FileOptions{GoPackage: proto.String("fireproto/test;fireprototest")}, + MessageType: []*descriptorpb.DescriptorProto{ + { + Name: proto.String("Tx"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("hash"), + Number: proto.Int32(1), + Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + JsonName: proto.String("hash"), + }, + { + Name: proto.String("fee"), + Number: proto.Int32(2), + Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + JsonName: proto.String("fee"), + Options: func() *descriptorpb.FieldOptions { + opts := &descriptorpb.FieldOptions{} + proto.SetExtension(opts, pbfirehose.E_Nondeterministic, true) + return opts + }(), + }, + }, + }, + }, + } + + blockFDP := &descriptorpb.FileDescriptorProto{ + Name: proto.String("fireproto_test_block.proto"), + Syntax: proto.String("proto3"), + Package: proto.String("fireproto.test"), + Dependency: []string{"fireproto_test_tx.proto"}, + Options: &descriptorpb.FileOptions{GoPackage: proto.String("fireproto/test;fireprototest")}, + MessageType: []*descriptorpb.DescriptorProto{ + { + Name: proto.String("Block"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("transactions"), + Number: proto.Int32(1), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(), + TypeName: proto.String(".fireproto.test.Tx"), + JsonName: proto.String("transactions"), + Options: func() *descriptorpb.FieldOptions { + opts := &descriptorpb.FieldOptions{} + proto.SetExtension(opts, pbfirehose.E_Transactions, true) + return opts + }(), + }, + { + Name: proto.String("gas"), + Number: proto.Int32(2), + Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + JsonName: proto.String("gas"), + Options: func() *descriptorpb.FieldOptions { + opts := &descriptorpb.FieldOptions{} + proto.SetExtension(opts, pbfirehose.E_Nondeterministic, true) + return opts + }(), + }, + }, + }, + }, + } + + files, err := protodesc.NewFiles(&descriptorpb.FileDescriptorSet{ + File: []*descriptorpb.FileDescriptorProto{txFDP, blockFDP}, + }) + require.NoError(t, err) + + txDesc, err := files.FindDescriptorByName("fireproto.test.Tx") + require.NoError(t, err) + blockDesc, err := files.FindDescriptorByName("fireproto.test.Block") + require.NoError(t, err) + + return dynamicpb.NewMessageType(blockDesc.(protoreflect.MessageDescriptor)), + dynamicpb.NewMessageType(txDesc.(protoreflect.MessageDescriptor)) +} + +func TestFindTransactionsField(t *testing.T) { + blockType, _ := buildAnnotatedDescriptors(t) + block := blockType.New().Interface() + + fd := fireproto.FindTransactionsField(block) + require.NotNil(t, fd) + assert.Equal(t, protoreflect.Name("transactions"), fd.Name()) +} + +func TestFindTransactionsField_none(t *testing.T) { + ts := ×tamppb.Timestamp{Seconds: 1} + assert.Nil(t, fireproto.FindTransactionsField(ts)) +} + +func TestWalkNonDeterministicFields(t *testing.T) { + blockType, txType := buildAnnotatedDescriptors(t) + + block := blockType.New() + tx1 := txType.New() + tx1.Set(tx1.Descriptor().Fields().ByName("hash"), protoreflect.ValueOfString("0xabc")) + tx1.Set(tx1.Descriptor().Fields().ByName("fee"), protoreflect.ValueOfBytes([]byte{1, 2, 3})) + block.Mutable(block.Descriptor().Fields().ByName("transactions")).List().Append(protoreflect.ValueOfMessage(tx1)) + block.Set(block.Descriptor().Fields().ByName("gas"), protoreflect.ValueOfBytes([]byte{0xff})) + + type result struct{ parent, field string } + var got []result + fireproto.WalkNonDeterministicFields(block.Interface(), func(msg protoreflect.Message, field protoreflect.FieldDescriptor) { + got = append(got, result{string(msg.Descriptor().Name()), string(field.Name())}) + }) + + assert.ElementsMatch(t, []result{ + {parent: "Block", field: "gas"}, + {parent: "Tx", field: "fee"}, + }, got) +} + +func TestWalkNonDeterministicFields_noAnnotations(t *testing.T) { + ts := ×tamppb.Timestamp{Seconds: 100} + var visited []string + fireproto.WalkNonDeterministicFields(ts, func(msg protoreflect.Message, field protoreflect.FieldDescriptor) { + visited = append(visited, string(field.Name())) + }) + assert.Empty(t, visited) +} + +func TestClearNonDeterministicFields(t *testing.T) { + blockType, txType := buildAnnotatedDescriptors(t) + + block := blockType.New() + tx1 := txType.New() + tx1.Set(tx1.Descriptor().Fields().ByName("fee"), protoreflect.ValueOfBytes([]byte{1, 2, 3})) + block.Mutable(block.Descriptor().Fields().ByName("transactions")).List().Append(protoreflect.ValueOfMessage(tx1)) + block.Set(block.Descriptor().Fields().ByName("gas"), protoreflect.ValueOfBytes([]byte{0xff})) + + fireproto.ClearNonDeterministicFields(block.Interface()) + + assert.False(t, block.Has(block.Descriptor().Fields().ByName("gas")), "gas should be cleared") + tx := block.Get(block.Descriptor().Fields().ByName("transactions")).List().Get(0).Message() + assert.False(t, tx.Has(tx.Descriptor().Fields().ByName("fee")), "fee should be cleared") + // hash was never set, just verify it's still unset + assert.False(t, tx.Has(tx.Descriptor().Fields().ByName("hash"))) +} diff --git a/go.mod b/go.mod index 196d4ef..e612359 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/streamingfast/logging v1.2.2 github.com/streamingfast/payment-gateway v0.0.0-20260527144655-d0576d2a4ee3 github.com/streamingfast/pbgo v0.0.6-0.20260206150405-2b95acf70437 + github.com/streamingfast/protox v0.0.0-00010101000000-000000000000 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 github.com/streamingfast/substreams v1.18.6-0.20260514140025-e5ca79eade0d github.com/stretchr/testify v1.11.1 From 9e27b5c0ca491fe6cc682a0faa595d8a450cd4af Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:36:06 -0400 Subject: [PATCH 5/8] chore: update CHANGELOG with fireproto package addition --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 957ac7c..06d4667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s ## Unreleased +### Added + +- New `fireproto` package with `WalkNonDeterministicFields`, `ClearNonDeterministicFields`, and `FindTransactionsField` utilities for walking and manipulating proto fields annotated with the `(firehose.nondeterministic)` and `(firehose.transactions)` field extensions. + ### Changed - `reader-node-firehose`: if the persisted cursor in the state file points to a block older than `--reader-node-start-block-num`, the cursor is now discarded (with a warning log) and the syncer restarts from the configured start block. Previously the stale cursor was always honored. From 526d40f00bd6d67b6bcee19fd0e7d93b85ce2f32 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:45:17 -0400 Subject: [PATCH 6/8] fix: improve fireproto doc comments and use Go 1.26 new() in tests --- fireproto/walker.go | 9 ++++++++ fireproto/walker_test.go | 46 ++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/fireproto/walker.go b/fireproto/walker.go index 52aa6fa..8c44a0f 100644 --- a/fireproto/walker.go +++ b/fireproto/walker.go @@ -10,6 +10,13 @@ import ( // WalkNonDeterministicFields walks the proto.Message tree rooted at root and calls fn // for each field tagged with (firehose.nondeterministic) = true. // fn receives the containing message instance and the field descriptor. +// +// fn is called for every non-deterministic field in the message type regardless of +// whether the field is currently populated in the instance; clearing an unset field +// is a no-op, so this is safe to use with ClearNonDeterministicFields. +// +// Well-known Google types (google.protobuf.*) are not recursed into. +// Self-referential message types are handled via DFS cycle prevention in the walker. func WalkNonDeterministicFields(root proto.Message, fn func(msg protoreflect.Message, field protoreflect.FieldDescriptor)) { for msg, field := range protox.WalkMessageInstanceFields(root.ProtoReflect(), nil) { if isNonDeterministic(field) { @@ -28,6 +35,8 @@ func ClearNonDeterministicFields(root proto.Message) { // FindTransactionsField returns the first field descriptor in root's message type // tagged with (firehose.transactions) = true, or nil if none is tagged. +// Only the top-level fields of root's message type are examined; nested messages +// are not searched. func FindTransactionsField(root proto.Message) protoreflect.FieldDescriptor { fields := root.ProtoReflect().Descriptor().Fields() for i := range fields.Len() { diff --git a/fireproto/walker_test.go b/fireproto/walker_test.go index 8ad28ed..c3f1b61 100644 --- a/fireproto/walker_test.go +++ b/fireproto/walker_test.go @@ -23,27 +23,27 @@ func buildAnnotatedDescriptors(t *testing.T) (blockType protoreflect.MessageType t.Helper() txFDP := &descriptorpb.FileDescriptorProto{ - Name: proto.String("fireproto_test_tx.proto"), - Syntax: proto.String("proto3"), - Package: proto.String("fireproto.test"), - Options: &descriptorpb.FileOptions{GoPackage: proto.String("fireproto/test;fireprototest")}, + Name: new("fireproto_test_tx.proto"), + Syntax: new("proto3"), + Package: new("fireproto.test"), + Options: &descriptorpb.FileOptions{GoPackage: new("fireproto/test;fireprototest")}, MessageType: []*descriptorpb.DescriptorProto{ { - Name: proto.String("Tx"), + Name: new("Tx"), Field: []*descriptorpb.FieldDescriptorProto{ { - Name: proto.String("hash"), - Number: proto.Int32(1), + Name: new("hash"), + Number: new(int32(1)), Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), - JsonName: proto.String("hash"), + JsonName: new("hash"), }, { - Name: proto.String("fee"), - Number: proto.Int32(2), + Name: new("fee"), + Number: new(int32(2)), Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), - JsonName: proto.String("fee"), + JsonName: new("fee"), Options: func() *descriptorpb.FieldOptions { opts := &descriptorpb.FieldOptions{} proto.SetExtension(opts, pbfirehose.E_Nondeterministic, true) @@ -56,22 +56,22 @@ func buildAnnotatedDescriptors(t *testing.T) (blockType protoreflect.MessageType } blockFDP := &descriptorpb.FileDescriptorProto{ - Name: proto.String("fireproto_test_block.proto"), - Syntax: proto.String("proto3"), - Package: proto.String("fireproto.test"), + Name: new("fireproto_test_block.proto"), + Syntax: new("proto3"), + Package: new("fireproto.test"), Dependency: []string{"fireproto_test_tx.proto"}, - Options: &descriptorpb.FileOptions{GoPackage: proto.String("fireproto/test;fireprototest")}, + Options: &descriptorpb.FileOptions{GoPackage: new("fireproto/test;fireprototest")}, MessageType: []*descriptorpb.DescriptorProto{ { - Name: proto.String("Block"), + Name: new("Block"), Field: []*descriptorpb.FieldDescriptorProto{ { - Name: proto.String("transactions"), - Number: proto.Int32(1), + Name: new("transactions"), + Number: new(int32(1)), Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(), - TypeName: proto.String(".fireproto.test.Tx"), - JsonName: proto.String("transactions"), + TypeName: new(".fireproto.test.Tx"), + JsonName: new("transactions"), Options: func() *descriptorpb.FieldOptions { opts := &descriptorpb.FieldOptions{} proto.SetExtension(opts, pbfirehose.E_Transactions, true) @@ -79,11 +79,11 @@ func buildAnnotatedDescriptors(t *testing.T) (blockType protoreflect.MessageType }(), }, { - Name: proto.String("gas"), - Number: proto.Int32(2), + Name: new("gas"), + Number: new(int32(2)), Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), - JsonName: proto.String("gas"), + JsonName: new("gas"), Options: func() *descriptorpb.FieldOptions { opts := &descriptorpb.FieldOptions{} proto.SetExtension(opts, pbfirehose.E_Nondeterministic, true) From dba3b1c71ebfeac291cc16aa923b99e1ece88283 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 10:46:36 -0400 Subject: [PATCH 7/8] chore: add pb/firehose/options.proto entry to CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06d4667..23af698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s ### Added -- New `fireproto` package with `WalkNonDeterministicFields`, `ClearNonDeterministicFields`, and `FindTransactionsField` utilities for walking and manipulating proto fields annotated with the `(firehose.nondeterministic)` and `(firehose.transactions)` field extensions. +- Add `pb/firehose/options.proto` defining `(firehose.transactions)` and `(firehose.nondeterministic)` custom `FieldOptions` extensions for annotating Block proto fields with semantic meaning (transaction list and non-deterministic values). +- Add `fireproto` package providing `WalkNonDeterministicFields`, `ClearNonDeterministicFields`, and `FindTransactionsField` utilities for walking and manipulating proto fields annotated with the `(firehose.nondeterministic)` and `(firehose.transactions)` field extensions. ### Changed From ecef69c83aa9158cec1a0d14b5c1b9c7f61bc2c9 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 10 Jun 2026 19:12:19 -0400 Subject: [PATCH 8/8] chore: remove local replace directive, pin protox to published commit --- go.mod | 3 +-- go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e612359..186f7f4 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/streamingfast/logging v1.2.2 github.com/streamingfast/payment-gateway v0.0.0-20260527144655-d0576d2a4ee3 github.com/streamingfast/pbgo v0.0.6-0.20260206150405-2b95acf70437 - github.com/streamingfast/protox v0.0.0-00010101000000-000000000000 + github.com/streamingfast/protox v0.0.0-20260610151242-cd8a8cf98c45 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 github.com/streamingfast/substreams v1.18.6-0.20260514140025-e5ca79eade0d github.com/stretchr/testify v1.11.1 @@ -312,6 +312,5 @@ replace ( cloud.google.com/go => github.com/streamingfast/google-cloud-go v0.0.0-20241202194114-f77ff78d4f66 github.com/ShinyTrinkets/overseer => github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef github.com/jhump/protoreflect => github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d - github.com/streamingfast/protox => /Users/maoueh/work/sf/protox github.com/tetratelabs/wazero => github.com/streamingfast/wazero v0.0.0-20241202185309-91287c3640ed ) diff --git a/go.sum b/go.sum index 7b9e54b..a83ca63 100644 --- a/go.sum +++ b/go.sum @@ -2357,6 +2357,8 @@ github.com/streamingfast/pbgo v0.0.6-0.20260206150405-2b95acf70437 h1:BsGEgwjlIJ github.com/streamingfast/pbgo v0.0.6-0.20260206150405-2b95acf70437/go.mod h1:MHb9nR5zyNyvA1Zi2wud+hWpoILpll7vpLUijvh3YV0= github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d h1:33VIARqUqBUKXJcuQoOS1rVSms54tgxhhNCmrLptpLg= github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d/go.mod h1:aBJivEdekmFWYSQ29EE/fN9IanJWJXbtjy3ky0XD/jE= +github.com/streamingfast/protox v0.0.0-20260610151242-cd8a8cf98c45 h1:hVLYp14KlfW9mnpNQoTXbHExUWHrX0A7Ey6z/veC5IE= +github.com/streamingfast/protox v0.0.0-20260610151242-cd8a8cf98c45/go.mod h1:/9DK62G+3E3hKwSMCs9vc+sNCDJBnhCxbb+KnZcIRDQ= github.com/streamingfast/sf-tracing v0.0.0-20251218140752-bafd5572499f h1:KXE5MEXwc7l6FwD9dBYH0r+0IojAS9VYZa/vuzCBnK0= github.com/streamingfast/sf-tracing v0.0.0-20251218140752-bafd5572499f/go.mod h1:H57LMxdkHi0MZ+n7xr7zo9PJv7kZZ38du9w0SizLcFU= github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAtyaTOgs=