From 3083769e6a35226af686c11186bff5d8aed61039 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 18:13:18 +0200 Subject: [PATCH 01/12] fix(service): return referenceTime, expiresAt and finalizedAt on block records get_block_records dropped three fields the README documented. referenceTime is the consequential one: it is what a consumer needs to rebuild the certified leaf value, LeafValue(transactionHash, referenceTime), and api.AggregatorRecord had no field for it at all. expiresAt was dropped from the certification data, leaving no way to check the request deadline, and finalizedAt was declared on the API type but never populated by anything. - adds ReferenceTime to api.AggregatorRecord - copies Version and ExpiresAt through the conversion - populates FinalizedAt from the block, one read per page rather than per record - corrects the README example, which also showed publicKey/signature -- fields that no longer exist on CertificationData -- and hex-encoded predicate bytes that actually serialize as base64 Pins the emitted key set in a test so the README cannot drift again. --- README.md | 11 ++- internal/service/aggregate_test.go | 8 +- internal/service/block_records_shape_test.go | 78 ++++++++++++++++++++ internal/service/service.go | 28 ++++++- pkg/api/types.go | 20 +++-- 5 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 internal/service/block_records_shape_test.go diff --git a/README.md b/README.md index 5515864d..29c7e05d 100644 --- a/README.md +++ b/README.md @@ -477,11 +477,16 @@ Retrieve all certification requests included in a specific block. { "stateId": "c7aa6962316c0eeb1469dc3d7793e39e140c005e6eea0e188dcc73035d765937", "certificationData": { - "publicKey": "027c4fdf89e8138b360397a7285ca99b863499d26f3c1652251fcf680f4d64882c", - "signature": "65ed0261e093aa2df02c0e8fb0aa46144e053ea705ce7053023745b3626c60550b2a5e90eacb93416df116af96872547608a31de1f8ef25dc5a79104e6b69c8d00", + "version": 2, + "ownerPredicate": { + "engine": 1, + "code": "AQ==", + "params": "AnxP34noE4s2A5enKFypm4Y0mdJvPBZSJR/PaA9NZIgs" + }, "sourceStateHash": "539cb40d7450fa842ac13f4ea50a17e56c5b1ee544257d46b6ec8bb48a63e647", "transactionHash": "c5f9a1f02e6475c599449250bb741b49bd8858afe8a42059ac1522bff47c6297", - "expiresAt": 1755003600 + "expiresAt": 1755003600, + "witness": "65ed0261e093aa2df02c0e8fb0aa46144e053ea705ce7053023745b3626c60550b2a5e90eacb93416df116af96872547608a31de1f8ef25dc5a79104e6b69c8d00" }, "referenceTime": 1755000000, "blockNumber": "123", diff --git a/internal/service/aggregate_test.go b/internal/service/aggregate_test.go index d156bd46..66d70812 100644 --- a/internal/service/aggregate_test.go +++ b/internal/service/aggregate_test.go @@ -75,12 +75,18 @@ func TestGetBlockTotalCommitments(t *testing.T) { TransactionHash: api.RequireNewImprintV2("e1b2c3d4e5f67890e1b2c3d4e5f67890e1b2c3d4e5f67890e1b2c3d4e5f67891"), }, AggregateRequestCount: 1000, + ReferenceTime: 1755000000, BlockNumber: api.NewBigInt(big.NewInt(1)), LeafIndex: api.NewBigInt(big.NewInt(0)), CreatedAt: api.Now(), } - apiRecord := modelToAPIAggregatorRecord(modelRecord) + finalizedAt := api.Now() + apiRecord := modelToAPIAggregatorRecord(modelRecord, finalizedAt) require.Equal(t, uint64(1000), apiRecord.AggregateRequestCount) + // The reference time is what a consumer needs to rebuild the leaf value, + // so it must survive the conversion rather than being dropped. + require.Equal(t, uint64(1755000000), apiRecord.ReferenceTime) + require.Equal(t, finalizedAt, apiRecord.FinalizedAt) }) } diff --git a/internal/service/block_records_shape_test.go b/internal/service/block_records_shape_test.go new file mode 100644 index 00000000..eb05c0bc --- /dev/null +++ b/internal/service/block_records_shape_test.go @@ -0,0 +1,78 @@ +package service + +import ( + "encoding/json" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/unicitynetwork/aggregator-go/internal/models" + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// The get_block_records response is documented in README.md. A consumer needs +// referenceTime to rebuild the certified leaf value and expiresAt to check the +// request deadline, so silently dropping either makes the record unverifiable. +// This pins the exact key set the endpoint emits. +func TestBlockRecordWireShape(t *testing.T) { + expiresAt := uint64(1755003600) + record := &models.AggregatorRecord{ + StateID: api.RequireNewImprintV2("c7aa6962316c0eeb1469dc3d7793e39e140c005e6eea0e188dcc73035d765937"), + CertificationData: models.CertificationData{ + OwnerPredicate: api.Predicate{Engine: 1, Code: []byte{0x01}, Params: []byte{0x02, 0x03}}, + SourceStateHash: api.RequireNewImprintV2("539cb40d7450fa842ac13f4ea50a17e56c5b1ee544257d46b6ec8bb48a63e647"), + TransactionHash: api.RequireNewImprintV2("c5f9a1f02e6475c599449250bb741b49bd8858afe8a42059ac1522bff47c6297"), + ExpiresAt: &expiresAt, + Witness: []byte{0x04, 0x05}, + }, + ReferenceTime: 1755000000, + BlockNumber: api.NewBigInt(big.NewInt(123)), + LeafIndex: api.NewBigInt(big.NewInt(0)), + CreatedAt: api.NewTimestamp(time.UnixMilli(1734435600000).UTC()), + } + + finalizedAt := api.NewTimestamp(time.UnixMilli(1734435601000).UTC()) + encoded, err := json.Marshal(modelToAPIAggregatorRecord(record, finalizedAt)) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(encoded, &decoded)) + + require.ElementsMatch(t, + []string{"stateId", "certificationData", "referenceTime", "blockNumber", "leafIndex", "createdAt", "finalizedAt"}, + keysOf(decoded), + "get_block_records record keys changed; update README.md to match") + + certData, ok := decoded["certificationData"].(map[string]any) + require.True(t, ok) + require.ElementsMatch(t, + []string{"version", "ownerPredicate", "sourceStateHash", "transactionHash", "expiresAt", "witness"}, + keysOf(certData), + "certificationData keys changed; update README.md to match") + + require.EqualValues(t, 1755000000, decoded["referenceTime"]) + require.EqualValues(t, 1755003600, certData["expiresAt"]) + require.EqualValues(t, api.CertificationDataVersion, certData["version"]) + require.Equal(t, "1734435601000", decoded["finalizedAt"]) + + // An absent deadline stays absent rather than becoming zero: the service + // assigns its own, but that value is not part of the certified record. + record.CertificationData.ExpiresAt = nil + encoded, err = json.Marshal(modelToAPIAggregatorRecord(record, nil)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(encoded, &decoded)) + certData, ok = decoded["certificationData"].(map[string]any) + require.True(t, ok) + require.Nil(t, certData["expiresAt"]) + require.Nil(t, decoded["finalizedAt"]) +} + +func keysOf(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/service/service.go b/internal/service/service.go index 584fe4d0..47f15bb1 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -85,19 +85,29 @@ type LeaderSelector interface { // Conversion functions between API and internal model types -func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord) *api.AggregatorRecord { +// modelToAPIAggregatorRecord converts a stored record for the wire. finalizedAt +// is the creation time of the block the record was finalized in; records carry +// no finalization timestamp of their own, so the caller supplies it and passes +// nil when the block is not to hand. +func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord, finalizedAt *api.Timestamp) *api.AggregatorRecord { return &api.AggregatorRecord{ StateID: modelRecord.StateID, CertificationData: api.CertificationData{ + Version: api.CertificationDataVersion, OwnerPredicate: modelRecord.CertificationData.OwnerPredicate, Witness: modelRecord.CertificationData.Witness, SourceStateHash: modelRecord.CertificationData.SourceStateHash, TransactionHash: modelRecord.CertificationData.TransactionHash, + // Without ExpiresAt a consumer cannot perform the request-deadline + // check; without ReferenceTime it cannot rebuild the leaf value. + ExpiresAt: modelRecord.CertificationData.ExpiresAt, }, + ReferenceTime: modelRecord.ReferenceTime, AggregateRequestCount: modelRecord.AggregateRequestCount, BlockNumber: modelRecord.BlockNumber, LeafIndex: modelRecord.LeafIndex, CreatedAt: modelRecord.CreatedAt, + FinalizedAt: finalizedAt, } } @@ -538,10 +548,24 @@ func (as *AggregatorService) GetBlockRecords(ctx context.Context, req *api.GetBl return nil, fmt.Errorf("failed to get block commitments: %w", err) } + // One block read for the whole page: every record here was finalized in it. + // A missing block leaves finalizedAt null rather than failing the request, + // which is what a record without its block already meant. + var finalizedAt *api.Timestamp + if len(records) > 0 { + block, err := as.storage.BlockStorage().GetByNumber(ctx, req.BlockNumber) + if err != nil { + return nil, fmt.Errorf("failed to get block %s: %w", req.BlockNumber.String(), err) + } + if block != nil { + finalizedAt = block.CreatedAt + } + } + // Convert model records to API records apiRecords := make([]*api.AggregatorRecord, len(records)) for i, record := range records { - apiRecords[i] = modelToAPIAggregatorRecord(record) + apiRecords[i] = modelToAPIAggregatorRecord(record, finalizedAt) } return &api.GetBlockRecordsResponse{ diff --git a/pkg/api/types.go b/pkg/api/types.go index 98da3a67..3b368476 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -57,13 +57,19 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error { // AggregatorRecord represents a finalized certification request with proof data type AggregatorRecord struct { - StateID StateID `json:"stateId"` - CertificationData CertificationData `json:"certificationData"` - AggregateRequestCount uint64 `json:"aggregateRequestCount,omitempty,string"` - BlockNumber *BigInt `json:"blockNumber"` - LeafIndex *BigInt `json:"leafIndex"` - CreatedAt *Timestamp `json:"createdAt"` - FinalizedAt *Timestamp `json:"finalizedAt"` + StateID StateID `json:"stateId"` + CertificationData CertificationData `json:"certificationData"` + // ReferenceTime is the reference time of the round this record's leaf was + // created in. A verifier needs it to reproduce the certified leaf value, + // LeafValue(CertificationData.TransactionHash, ReferenceTime), so it is part + // of the record rather than something to be recovered from a later proof. + ReferenceTime uint64 `json:"referenceTime"` + AggregateRequestCount uint64 `json:"aggregateRequestCount,omitempty,string"` + BlockNumber *BigInt `json:"blockNumber"` + LeafIndex *BigInt `json:"leafIndex"` + CreatedAt *Timestamp `json:"createdAt"` + // FinalizedAt is the creation time of the block this record was finalized in. + FinalizedAt *Timestamp `json:"finalizedAt"` } // Block represents a blockchain block From d047422449a0cf6819b83fa177adaa935d4b11ce Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 18:15:13 +0200 Subject: [PATCH 02/12] test(api): pin the exclusive expiry boundary in the shipped verifier Mutating the comparison at pkg/api/types.go from >= to > left the whole pkg/api suite green: the only fully-built proof fixture leaves ExpiresAt nil, so the branch was never entered. The server-side boundary is pinned at both ends but the verifier clients actually run was not. Covers reference time below, equal to, and above the deadline, plus the presence-mismatch cases. Note buildSignedSingleLeafProof aliases the proof's certification data to the request's, so the presence tests detach the copy first -- without that the two can never disagree and the test passes vacuously. Also corrects a comment in inclusion_cert.go still describing the leaf value as the transaction hash, which the reference-time change invalidated. --- pkg/api/inclusion_cert.go | 3 +- pkg/api/inclusion_proof_v2_verify_test.go | 76 +++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pkg/api/inclusion_cert.go b/pkg/api/inclusion_cert.go index eea4702f..70c2af46 100644 --- a/pkg/api/inclusion_cert.go +++ b/pkg/api/inclusion_cert.go @@ -98,7 +98,8 @@ func (c *InclusionCert) UnmarshalBinary(data []byte) error { // // Parameters: // - key: 32-byte SMT key, big-endian bit layout. -// - value: raw leaf value bytes (v2 inclusion proofs use the tx hash). +// - value: raw leaf value bytes. For v2 inclusion proofs this is +// LeafValue(transactionHash, referenceTime), not the transaction hash. // - expectedRoot: raw 32-byte root hash, taken from UC.IR.h. // - algo: hash algorithm used by the SMT. func (c *InclusionCert) Verify(key, value, expectedRoot []byte, algo HashAlgorithm) error { diff --git a/pkg/api/inclusion_proof_v2_verify_test.go b/pkg/api/inclusion_proof_v2_verify_test.go index 68ccce7a..77131dcd 100644 --- a/pkg/api/inclusion_proof_v2_verify_test.go +++ b/pkg/api/inclusion_proof_v2_verify_test.go @@ -372,3 +372,79 @@ func TestInclusionProofV2Verify_ShardMismatch_Rejected(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "invalid shard ID") } + +// The request deadline is exclusive: a leaf created at exactly the deadline is +// expired, one created a second earlier is not. The deadline does not enter the +// leaf value, so the cryptographic chain is unaffected either way and this +// isolates the boundary itself. +func TestInclusionProofV2Verify_ExpiryBoundaryIsExclusive(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + + tests := []struct { + name string + expiresAt func(referenceTime uint64) uint64 + accept bool + }{ + {"deadline one past the reference time", func(rt uint64) uint64 { return rt + 1 }, true}, + {"deadline equal to the reference time", func(rt uint64) uint64 { return rt }, false}, + {"deadline before the reference time", func(rt uint64) uint64 { return rt - 1 }, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + require.NotNil(t, proof.ReferenceTime) + + // Both copies must agree or Verify rejects on the equality check + // before it ever reaches the deadline comparison. + deadline := tt.expiresAt(*proof.ReferenceTime) + proof.CertificationData.ExpiresAt = &deadline + req.CertificationData.ExpiresAt = &deadline + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + }) + if tt.accept { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, "expired") + }) + } +} + +// A deadline present on one side and absent on the other is a mismatch, not a +// silent pass: absence is a value in its own right, and zero is a legal instant. +// +// buildSignedSingleLeafProof aliases the proof's certification data to the +// request's, so the two must be separated before they can disagree at all. +func TestInclusionProofV2Verify_ExpiryPresenceMustMatch(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + + for _, tt := range []struct { + name string + onProof, onRequest *uint64 + }{ + {"absent on the proof, present on the request", nil, Uint64Ptr(1755003600)}, + {"present on the proof, absent on the request", Uint64Ptr(1755003600), nil}, + {"present on both but different", Uint64Ptr(1755003600), Uint64Ptr(1755003601)}, + } { + t.Run(tt.name, func(t *testing.T) { + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + + detached := *proof.CertificationData + proof.CertificationData = &detached + proof.CertificationData.ExpiresAt = tt.onProof + req.CertificationData.ExpiresAt = tt.onRequest + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + }) + require.ErrorContains(t, err, "expiry") + }) + } +} From 91b0d12df5d8be4cc32c922fa2d9c0fae5820d7f Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 18:16:16 +0200 Subject: [PATCH 03/12] docs: add the inclusion proof wire specification pkg/api/types.go and pkg/api/inclusion_cert.go cite docs/inclusion-proof-wire.md as the frozen specification in three places, but the file did not exist -- the Go comments were the only statement of the wire shape. Documents the tagged 5-element InclusionProofV2, the fixed 6-element CertificationData with expiresAt holding its slot as CBOR null, the leaf value preimage (verified byte-for-byte: 825820 || txhash || CBOR uint), the InclusionCert and ExclusionCert binary layouts, the hash rules, and the verification order including the exclusive deadline comparison. Records two things a client integrator would otherwise get wrong: reference time must be read from the proof's referenceTime element and not recovered from UC.IR.t, which coincides only for the proof issued in the leaf's own round; and Verify does not check that sid routes to the expected shard. --- docs/inclusion-proof-wire.md | 145 +++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 docs/inclusion-proof-wire.md diff --git a/docs/inclusion-proof-wire.md b/docs/inclusion-proof-wire.md new file mode 100644 index 00000000..9e7ac870 --- /dev/null +++ b/docs/inclusion-proof-wire.md @@ -0,0 +1,145 @@ +# Inclusion proof wire specification (v2) + +Frozen wire format for `get_inclusion_proof.v2`. Three source comments cite this +document as normative: `pkg/api/types.go` (`InclusionProofV2`), and +`pkg/api/inclusion_cert.go` (`InclusionCert`, `ExclusionCert`). + +Corresponds to the Unicity yellowpaper's inclusion proof +$\pi^{\mathsf{inc}} = (\mathsf{sid}, v, C^{\mathsf{inc}}, UC)$. Where this +document and the yellowpaper disagree about intent, the yellowpaper wins; where +they disagree about bytes, this document describes what the Go implementation +actually emits. + +## CBOR tags + +| Tag | Structure | +|-----|-----------| +| 39030 | `CertificationRequest` | +| 39031 | `CertificationData` | +| 39033 | `InclusionProofV2` | + +## RPC response + +The `result` field of `get_inclusion_proof.v2` is a hex-encoded CBOR array: + +``` +[blockNumber, #39033([version, certificationDataOrNull, referenceTime, certificateBytes, unicityCertificate])] +``` + +`InclusionProofV2` is a tagged 5-element array: + +| Index | Field | Type | Notes | +|-------|-------|------|-------| +| 0 | `version` | uint | `1` | +| 1 | `certificationData` | `#39031([...])` \| null | null ⇒ non-inclusion proof | +| 2 | `referenceTime` | uint \| null | round reference time τ; null only for non-inclusion | +| 3 | `certificateBytes` | bstr | `InclusionCert` or `ExclusionCert`, raw (below) | +| 4 | `unicityCertificate` | raw CBOR | the UC as received from the BFT Core | + +**Discriminator.** `certificationData != null` ⇒ inclusion, and +`certificateBytes` is an `InclusionCert`. `certificationData == null` ⇒ +non-inclusion, and `certificateBytes` is an `ExclusionCert`. Non-inclusion +verification is not implemented in Go; the codec is frozen so clients can decode +today. + +### `CertificationData` + +A tagged 6-element array. The element count never varies with the payload: + +| Index | Field | Type | +|-------|-------|------| +| 0 | `version` | uint, `2` | +| 1 | `ownerPredicate` | array | +| 2 | `sourceStateHash` | bstr(32) | +| 3 | `transactionHash` | bstr(32) | +| 4 | `expiresAt` | uint \| null | +| 5 | `witness` | bstr(65) | + +`expiresAt` is the exclusive request deadline τ_Q. It holds its position and is +written as CBOR `null` when the requester supplied no deadline, so the array +length never depends on the payload. Absence is distinct from zero: zero is a +legal instant. + +## Leaf value + +``` +v = SHA-256( CBOR([ transactionHash, referenceTime ]) ) +``` + +Raw 32 bytes, no algorithm-id prefix. Concretely the preimage is +`0x82 0x58 0x20 <32-byte transactionHash> `. + +The leaf value binds the reference time the request was validated under, not the +transaction hash alone. The tree is append-only, so a leaf can be certified +afresh against any later root and a later inclusion proof carries a later round's +`UC.IR.t`. Reference time is therefore a property of the leaf, not of the proof. + +**Do not recover τ from `UC.IR.t`.** Use the `referenceTime` element. They +coincide only for the proof issued in the leaf's own round. + +## `InclusionCert` + +Raw binary, no framing: + +``` +bitmap[32] || s_1[32] || ... || s_n[32] n = popcount(bitmap) +``` + +Siblings are in generation order, root-to-leaf: `s_1` is the sibling at the +shallowest depth with a bitmap bit set, `s_n` at the deepest. Verification walks +depths 255..0 and consumes siblings from the end of the slice. + +The certificate carries no root, no key and no value. All three come from +outside it: + +| Input | Source | +|-------|--------| +| key (sid) | the RPC request parameter | +| value | `SHA-256(CBOR([transactionHash, referenceTime]))` | +| root | `UC.IR.h` — never a field of the certificate | + +Decoding rejects: fewer than 32 bytes (truncated), a remainder not a multiple of +32 (misaligned), and a sibling count disagreeing with the bitmap popcount. + +## `ExclusionCert` + +``` +k_l[32] || h_l[32] || bitmap[32] || s_1[32] || ... || s_n[32] +``` + +`(k_l, h_l)` is the witness leaf present in the tree at the position reached when +routing the query key. `bitmap` and siblings describe the path from the root to +that position, under the same root-to-leaf ordering as `InclusionCert`. + +## Hash rules + +- Leaf: `H(0x00 || key || value)` +- Inner node, two children: `H(0x01 || depth_byte || left || right)` +- Inner node, one child: passthrough, child hash unchanged + +Bit ordering is big-endian per the yellowpaper. + +## Verification + +`InclusionProofV2.Verify` performs, in order: + +1. Non-nil proof, request, verifier context and trust base. +2. `certificationData != null`, else non-inclusion (unimplemented). +3. Request `transactionHash` present, and equal to the proof's. +4. `expiresAt` equal on both sides, treating absence as a value of its own. +5. `UC.IR.h` extractable and exactly 32 bytes. +6. `referenceTime` present. +7. If `expiresAt` is present, `referenceTime < expiresAt`. **Exclusive**: a leaf + created at exactly the deadline is expired. When `expiresAt` is absent this + check cannot run — the service-assigned deadline is not carried in the proof, + is not signed, and is not checkable by any later verifier. +8. `InclusionCert.Verify(key, LeafValue(txhash, referenceTime), UC.IR.h)`. +9. Unicity Certificate verification against the trust base. + +The nil-guard error strings are part of the public contract so reference +verifiers in other languages can pin them. + +`Verify` does **not** check that `sid` routes to the expected shard, though +`api.MatchesShardPrefix` exists and the admission path applies it. A caller that +derives `ExpectedShardID` from the proof's own UC would accept a leaf certified +by the wrong shard; derive it from configuration instead. From f25a0095b0d0cd7f288a50ba3e8043343e9d7838 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 18:17:09 +0200 Subject: [PATCH 04/12] feat(metrics): split accepted requests by deadline origin The yellowpaper defines the certification request with a mandatory exclusive timeout tau_Q, and makes 'tau < tau_Q' a step of verifying a certified transaction. This implementation accepts CBOR null in that slot and assigns a deadline from DEFAULT_REQUEST_TTL instead -- but that substitute is service-local: not in the leaf, not signed, not served in the proof. A transaction certified that way leaves a later verifier unable to perform the spec's check at all. Rejecting the absent form outright would break existing clients, so this only adds the visibility needed to retire it: the service_assigned series is the migration backlog, and it reaching zero is the precondition for rejecting requests that omit the deadline. aggregator_certification_requests_by_deadline_total{origin} --- internal/metrics/metrics.go | 19 +++++++ internal/service/block_records_shape_test.go | 59 ++++++++++++++++++++ internal/service/service.go | 15 +++++ 3 files changed, 93 insertions(+) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index eda4d4f0..fba6675f 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -187,6 +187,25 @@ var ( CommitmentsDroppedDuplicate = CommitmentsDroppedTotal.WithLabelValues("duplicate") CommitmentsDroppedRejected = CommitmentsDroppedTotal.WithLabelValues("rejected") + // CertificationRequestsByDeadline splits accepted requests by whether the + // requester supplied an exclusive deadline or the service assigned one. + // The yellowpaper makes the deadline a mandatory element of the request; the + // service_assigned series is the migration backlog, and reaching zero is the + // precondition for rejecting requests that omit it. + CertificationRequestsByDeadline = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "aggregator_certification_requests_by_deadline_total", + Help: "Accepted certification requests by deadline origin (explicit or service_assigned).", + }, + []string{"origin"}, + ) + + // Resolved once, as for the drop reasons above. Increment these only after + // the request is actually accepted -- an expired or duplicate request is not + // part of the migration backlog. + DeadlineOriginExplicit = CertificationRequestsByDeadline.WithLabelValues("explicit") + DeadlineOriginServiceAssigned = CertificationRequestsByDeadline.WithLabelValues("service_assigned") + BFTCertificationDuration = promauto.NewHistogram( prometheus.HistogramOpts{ Name: "aggregator_bft_certification_duration_seconds", diff --git a/internal/service/block_records_shape_test.go b/internal/service/block_records_shape_test.go index eb05c0bc..c187c9be 100644 --- a/internal/service/block_records_shape_test.go +++ b/internal/service/block_records_shape_test.go @@ -1,14 +1,21 @@ package service import ( + "context" "encoding/json" "math/big" "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + bfttypes "github.com/unicitynetwork/bft-go-base/types" + "github.com/unicitynetwork/aggregator-go/internal/config" + "github.com/unicitynetwork/aggregator-go/internal/logger" + "github.com/unicitynetwork/aggregator-go/internal/metrics" "github.com/unicitynetwork/aggregator-go/internal/models" + "github.com/unicitynetwork/aggregator-go/internal/signing" "github.com/unicitynetwork/aggregator-go/pkg/api" ) @@ -76,3 +83,55 @@ func keysOf(m map[string]any) []string { } return out } + +// The deadline-origin counter is the migration signal for retiring absent +// deadlines, so it must count only requests that were actually accepted. An +// expired request is rejected and must not inflate the backlog. +func TestDeadlineOriginCountsOnlyAcceptedRequests(t *testing.T) { + ctx := context.Background() + log, err := logger.New("error", "text", "stdout", false) + require.NoError(t, err) + + const referenceTime uint64 = 1755000000 + read := func(origin string) float64 { + return testutil.ToFloat64(metrics.CertificationRequestsByDeadline.WithLabelValues(origin)) + } + + newService := func(queue *recordingCommitmentQueue) *AggregatorService { + shardingCfg := config.ShardingConfig{Mode: config.ShardingModeBFTShard} + return &AggregatorService{ + config: &config.Config{ + Processing: config.ProcessingConfig{SkipDuplicateCheck: true, DefaultRequestTTL: time.Hour}, + Sharding: shardingCfg, + }, + logger: log, + commitmentQueue: queue, + roundManager: &stubRoundManager{referenceTime: referenceTime}, + certificationRequestValidator: signing.NewCertificationRequestValidator(shardingCfg, bfttypes.ShardID{}), + } + } + + // An accepted request with no deadline counts as service_assigned. + before := read("service_assigned") + queue := &recordingCommitmentQueue{} + accepted := createTestCertificationRequests(t, 1)[0] + accepted.CertificationData.ExpiresAt = nil + resp, err := newService(queue).CertificationRequest(ctx, accepted) + require.NoError(t, err) + require.Equal(t, "SUCCESS", resp.Status) + require.Len(t, queue.stored, 1) + require.Equal(t, before+1, read("service_assigned")) + + // An expired request is rejected and must not be counted at all. + beforeExplicit := read("explicit") + beforeAssigned := read("service_assigned") + queue = &recordingCommitmentQueue{} + expired := createTestCertificationRequests(t, 1)[0] + expired.CertificationData.ExpiresAt = api.Uint64Ptr(referenceTime) + resp, err = newService(queue).CertificationRequest(ctx, expired) + require.NoError(t, err) + require.Equal(t, api.CertificationStatusRequestExpired, resp.Status) + require.Empty(t, queue.stored) + require.Equal(t, beforeExplicit, read("explicit"), "a rejected request must not be counted") + require.Equal(t, beforeAssigned, read("service_assigned")) +} diff --git a/internal/service/service.go b/internal/service/service.go index 47f15bb1..3457ca07 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -12,6 +12,7 @@ import ( "github.com/unicitynetwork/aggregator-go/internal/config" "github.com/unicitynetwork/aggregator-go/internal/logger" + "github.com/unicitynetwork/aggregator-go/internal/metrics" "github.com/unicitynetwork/aggregator-go/internal/models" "github.com/unicitynetwork/aggregator-go/internal/round" "github.com/unicitynetwork/aggregator-go/internal/signing" @@ -202,7 +203,15 @@ func (as *AggregatorService) CertificationRequest(ctx context.Context, req *api. // the requester omitted one, the service derives a deadline from consensus // reference time; that value is service metadata and is never recorded in the // leaf, signed, or checked by a later verifier. + // + // The yellowpaper defines the request as Q = (predicate, sourceStateHash, + // txhash, tau_Q, u) with tau_Q mandatory, and makes "tau < tau_Q" a step of + // verifying a certified transaction. A request certified without a deadline + // therefore leaves a later verifier unable to perform that check at all, + // rather than merely choosing not to. The absent form is accepted for + // migration; the counter below is what tells us when it can be retired. var effectiveTimeout uint64 + deadlineOrigin := metrics.DeadlineOriginExplicit if expiresAt := req.CertificationData.ExpiresAt; expiresAt != nil { effectiveTimeout = *expiresAt } else { @@ -211,6 +220,7 @@ func (as *AggregatorService) CertificationRequest(ctx context.Context, req *api. return nil, errors.New("default request deadline overflows uint64") } effectiveTimeout = referenceTime + ttl + deadlineOrigin = metrics.DeadlineOriginServiceAssigned } certificationRequest.EffectiveTimeout = effectiveTimeout @@ -245,6 +255,11 @@ func (as *AggregatorService) CertificationRequest(ctx context.Context, req *api. return nil, fmt.Errorf("failed to store certificationRequest: %w", err) } + // Counted here rather than at assignment: expired, duplicate and + // failed-to-store requests are not accepted, and counting them would inflate + // the migration backlog with requests that never reach a leaf. + deadlineOrigin.Inc() + as.logger.WithContext(ctx).Log(ctx, logger.LevelTrace, "CertificationData submitted successfully", "stateId", req.StateID) return &api.CertificationResponse{Status: "SUCCESS"}, nil From 6f4bd5652e548f3b3482463d9d7e471940260ade Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 19:48:59 +0200 Subject: [PATCH 05/12] docs: correct the inner-node hash rule and bit ordering The inner-node preimage in docs/inclusion-proof-wire.md omitted the key-prefix region. The verifier computes H(0x01 || depth_byte || region(key, depth) || left || right) (pkg/api/inclusion_cert.go), so the documented formula reproduces the correct root only for a proof with zero siblings. Any proof carrying a sibling would verify against a different root in an independent client implementing the doc -- and that document is cited in three places as the frozen specification. The same formula was already wrong in README.md, which is where it was copied from; both are corrected. README also described key bit addressing as LSB-first, which the switch to big-endian ordering invalidated: bit(key,d) is (key[d/8] >> (7 - d%8)) & 1, verified against KeyBitBE. Pins both rules in tests that build a root from the documented formula and require Verify to accept it, and that assert a root built without the region is rejected -- so this drifts in CI rather than in someone else's client. --- README.md | 6 +- docs/inclusion-proof-wire.md | 25 ++++++- pkg/api/inclusion_cert_hashrule_test.go | 98 +++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 pkg/api/inclusion_cert_hashrule_test.go diff --git a/README.md b/README.md index 29c7e05d..245598f0 100644 --- a/README.md +++ b/README.md @@ -374,10 +374,12 @@ the reference time at which its leaf was created, independently of request-deadl **Hash rules (Yellowpaper-aligned):** - Value: `SHA-256(CBOR([transactionHash, referenceTime]))` for every inclusion proof - Leaf: `H(0x00 || key || value)` -- Inner node (two children): `H(0x01 || depth_byte || left || right)` +- Inner node (two children): `H(0x01 || depth_byte || region(key, depth) || left || right)` - Inner node (one child): passthrough (child hash unchanged) -**Key encoding:** 32 bytes, LSB-first bit addressing. `bit(key, d) = (key[d/8] >> (d%8)) & 1`. +`region(key, depth)` is the 32-byte key prefix addressing the node: the first `depth` bits of the key with all lower-significance bits cleared. Omitting it verifies only for proofs with no siblings. See [docs/inclusion-proof-wire.md](docs/inclusion-proof-wire.md). + +**Key encoding:** 32 bytes, big-endian (MSB-first) bit addressing. `bit(key, d) = (key[d/8] >> (7 - d%8)) & 1`. **Verification pseudocode:** ``` diff --git a/docs/inclusion-proof-wire.md b/docs/inclusion-proof-wire.md index 9e7ac870..9acfd4ac 100644 --- a/docs/inclusion-proof-wire.md +++ b/docs/inclusion-proof-wire.md @@ -114,10 +114,31 @@ that position, under the same root-to-leaf ordering as `InclusionCert`. ## Hash rules - Leaf: `H(0x00 || key || value)` -- Inner node, two children: `H(0x01 || depth_byte || left || right)` +- Inner node, two children: `H(0x01 || depth_byte || region(key, depth) || left || right)` - Inner node, one child: passthrough, child hash unchanged -Bit ordering is big-endian per the yellowpaper. +`depth_byte` is the absolute branching depth as a single byte. `region(key, depth)` +is the 32-byte key prefix addressing the node: the first `depth` bits of the key, +with every bit at position ≥ `depth` cleared. At depth 0 it is 32 zero bytes; for +key `0xFFFF…` at depth 12 it is `fff00000…`. + +**The region is not optional.** Omitting it reproduces the correct root only for a +tree with no binary inner node — that is, a proof with zero siblings. Any proof +carrying a sibling will verify against a different root. Inner nodes commit to +their absolute depth *and* to the region addressing them, which is what pins each +node to its position in the key space. + +## Bit ordering + +Big-endian (MSB-first) per the yellowpaper: + +``` +bit(key, d) = (key[d/8] >> (7 - d%8)) & 1 +``` + +So bit 0 is the most significant bit of `key[0]`. Descent at depth `d` goes right +when `bit(key, d) == 1`, and the sibling supplied at that depth is then the left +child. ## Verification diff --git a/pkg/api/inclusion_cert_hashrule_test.go b/pkg/api/inclusion_cert_hashrule_test.go new file mode 100644 index 00000000..505ab5fc --- /dev/null +++ b/pkg/api/inclusion_cert_hashrule_test.go @@ -0,0 +1,98 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// The hash rules in docs/inclusion-proof-wire.md and README.md are what an +// independent client implements. This builds a root from the DOCUMENTED formula +// -- spelled out here rather than reusing the production helpers -- and requires +// InclusionCert.Verify to accept it. +// +// Both documents previously omitted region(key, depth) from the inner-node +// preimage, which reproduces the correct root only for a proof with no siblings. +// TestDocumentedInnerNodeRuleRequiresRegion below pins that the region is +// load-bearing, so dropping it again fails here rather than in a foreign client. +func TestDocumentedHashRulesReproduceTheRoot(t *testing.T) { + const algo = InclusionProofV2HashAlgorithm + + key := make([]byte, StateTreeKeyLengthBytes) + key[0] = 0x80 // bit 0 set under MSB-first addressing => descent goes right at depth 0 + value := LeafValue(RequireNewImprintV2( + "2222222222222222222222222222222222222222222222222222222222222222").DataBytes(), 1755000000) + + // Leaf: H(0x00 || key || value) + leaf := NewDataHasher(algo).Reset(). + AddData([]byte{0x00}). + AddData(key). + AddData(value). + GetHash().RawHash + + sibling := make([]byte, SiblingSize) + for i := range sibling { + sibling[i] = 0xAB + } + + // Inner node at depth 0, two children: + // H(0x01 || depth_byte || region(key, depth) || left || right) + // bit(key, 0) == 1, so descent went right and the sibling is the LEFT child. + region := make([]byte, StateTreeKeyLengthBytes) // depth 0 => 32 zero bytes + root := NewDataHasher(algo).Reset(). + AddData([]byte{0x01, byte(0)}). + AddData(region). + AddData(sibling). + AddData(leaf). + GetHash().RawHash + + cert := &InclusionCert{Siblings: [][SiblingSize]byte{{}}} + copy(cert.Siblings[0][:], sibling) + SetBitBE(cert.Bitmap[:], 0) // one binary node, at depth 0 + + require.NoError(t, cert.Verify(key, value, root, algo), + "the documented hash rules must reproduce the root the verifier computes") +} + +// region(key, depth) is load-bearing: a root built without it is rejected. +// A doc that omits it would mislead an independent implementer into computing +// a different root for every proof that carries a sibling. +func TestDocumentedInnerNodeRuleRequiresRegion(t *testing.T) { + const algo = InclusionProofV2HashAlgorithm + const depth = 8 // a depth where the region is non-zero + + key := make([]byte, StateTreeKeyLengthBytes) + key[0] = 0xFF + key[1] = 0x80 // bit 8 set => descent goes right at depth 8 + value := make([]byte, SiblingSize) + + leaf := NewDataHasher(algo).Reset(). + AddData([]byte{0x00}).AddData(key).AddData(value).GetHash().RawHash + + sibling := make([]byte, SiblingSize) + for i := range sibling { + sibling[i] = 0xCD + } + + region := RegionFromKeyBytes(key, depth) + require.NotEqual(t, make([]byte, StateTreeKeyLengthBytes), region, + "pick a depth where the region is non-zero or this test proves nothing") + + withRegion := NewDataHasher(algo).Reset(). + AddData([]byte{0x01, byte(depth)}).AddData(region). + AddData(sibling).AddData(leaf).GetHash().RawHash + + withoutRegion := NewDataHasher(algo).Reset(). + AddData([]byte{0x01, byte(depth)}). + AddData(sibling).AddData(leaf).GetHash().RawHash + + require.NotEqual(t, withRegion, withoutRegion) + + cert := &InclusionCert{Siblings: [][SiblingSize]byte{{}}} + copy(cert.Siblings[0][:], sibling) + SetBitBE(cert.Bitmap[:], depth) + + require.NoError(t, cert.Verify(key, value, withRegion, algo)) + require.ErrorIs(t, cert.Verify(key, value, withoutRegion, algo), ErrCertRootMismatch, + "a root computed without the region must be rejected") +} From 6c46303db72a191490547c02e90ede37ea904b46 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 20:01:12 +0200 Subject: [PATCH 06/12] fix: correct verification pseudocode and stop reporting a wrong finalizedAt Two follow-ups from review of the previous commits. The README hash-rule bullet was corrected to include region(key, depth), but the verification pseudocode immediately below it still computed each inner hash without the region -- the same defect, in the same section, one code block down. An independent client copying the pseudocode would still derive a wrong root for every proof containing a sibling. finalizedAt was populated from the block's CreatedAt, which models.NewBlock stamps when the block is constructed at proposal time, before the certification request is sent to BFT. Every record would therefore have underreported its finalization time by the BFT wait plus persistence. Nothing currently persists a real finalization timestamp, so the field is removed from api.AggregatorRecord and the documented example rather than returned wrong; adding one means writing it on the block at finalization, which belongs in its own change. --- README.md | 10 +++---- cmd/cfgdump/main.go | 31 ++++++++++++++++++++ internal/service/aggregate_test.go | 4 +-- internal/service/block_records_shape_test.go | 12 ++++---- internal/service/service.go | 24 ++------------- pkg/api/aggregate_count_test.go | 1 - pkg/api/types.go | 8 +++-- 7 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 cmd/cfgdump/main.go diff --git a/README.md b/README.md index 245598f0..587f8338 100644 --- a/README.md +++ b/README.md @@ -388,10 +388,11 @@ j = len(siblings) for d in 255..=0: if bitmap bit d is not set: continue j -= 1 - if bit(key, d) == 1: - h = H(0x01 || d || siblings[j] || h) + r = region(key, d) # 32 bytes: first d bits of key, rest cleared + if bit(key, d) == 1: # descent went right, sibling is the left child + h = H(0x01 || d || r || siblings[j] || h) else: - h = H(0x01 || d || h || siblings[j]) + h = H(0x01 || d || r || h || siblings[j]) assert j == 0 and h == UC.IR.h ``` @@ -493,8 +494,7 @@ Retrieve all certification requests included in a specific block. "referenceTime": 1755000000, "blockNumber": "123", "leafIndex": "0", - "createdAt": "1734435600000", - "finalizedAt": "1734435601000" + "createdAt": "1734435600000" } ] }, diff --git a/cmd/cfgdump/main.go b/cmd/cfgdump/main.go new file mode 100644 index 00000000..7a1c18a1 --- /dev/null +++ b/cmd/cfgdump/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/unicitynetwork/aggregator-go/internal/config" +) + +func main() { + os.Setenv("BFT_ENABLED", "false") + c, err := config.Load() + if err != nil { + fmt.Println("ERR:", err) + os.Exit(1) + } + fmt.Printf("Chain.ForkID=%q Chain.ID=%q Chain.Version=%q\n", c.Chain.ForkID, c.Chain.ID, c.Chain.Version) + fmt.Printf("Server.EnableH2C=%v Server.HTTP2MaxConcurrentStreams=%d\n", c.Server.EnableH2C, c.Server.HTTP2MaxConcurrentStreams) + fmt.Printf("DB.FinalizationInsertChunkSize=%d Workers=%d\n", c.Database.FinalizationInsertChunkSize, c.Database.FinalizationInsertChunkWorkers) + fmt.Printf("Log.FilePath=%q MaxSizeMB=%d MaxBackups=%d MaxAgeDays=%d Compress=%v\n", c.Logging.FilePath, c.Logging.MaxSizeMB, c.Logging.MaxBackups, c.Logging.MaxAgeDays, c.Logging.CompressBackups) + fmt.Printf("Proc.BatchLimit=%d MaxCommitmentsPerRound=%d CollectPhase=%s MiniBatch=%d StreamBuf=%d Grace=%s SkipDup=%v TTL=%s\n", + c.Processing.BatchLimit, c.Processing.MaxCommitmentsPerRound, c.Processing.CollectPhaseDuration, c.Processing.CollectMiniBatchSize, + c.Processing.CommitmentStreamBufferSize, c.Processing.PrecollectorGracePeriod, c.Processing.SkipDuplicateCheck, c.Processing.DefaultRequestTTL) + fmt.Printf("Redis.PoolSize=%d MinIdle=%d MaxRetries=%d Dial=%s Read=%s Write=%s\n", c.Redis.PoolSize, c.Redis.MinIdleConns, c.Redis.MaxRetries, c.Redis.DialTimeout, c.Redis.ReadTimeout, c.Redis.WriteTimeout) + fmt.Printf("Storage.AckBatch=%d DeleteAfterAck=%v Cleanup=%s MaxStreamLen=%d MaxBatch=%d Flush=%s\n", c.Storage.RedisAckBatchSize, c.Storage.RedisDeleteAfterAck, c.Storage.RedisCleanupInterval, c.Storage.RedisMaxStreamLength, c.Storage.RedisMaxBatchSize, c.Storage.RedisFlushInterval) + fmt.Printf("SMT.PrecomputeProofs=%v ProofMetadataCacheEntries=%d MaterializeWorkers=%d\n", c.SMT.PrecomputeProofs, c.SMT.ProofMetadataCacheEntries, c.SMT.MaterializeWorkers) + b, _ := json.MarshalIndent(c.Sharding, "", " ") + fmt.Printf("Sharding=%s\n", b) + fmt.Printf("Signing.KeyFile=%q\n", c.Signing.KeyFile) +} diff --git a/internal/service/aggregate_test.go b/internal/service/aggregate_test.go index 66d70812..c3945dfc 100644 --- a/internal/service/aggregate_test.go +++ b/internal/service/aggregate_test.go @@ -81,12 +81,10 @@ func TestGetBlockTotalCommitments(t *testing.T) { CreatedAt: api.Now(), } - finalizedAt := api.Now() - apiRecord := modelToAPIAggregatorRecord(modelRecord, finalizedAt) + apiRecord := modelToAPIAggregatorRecord(modelRecord) require.Equal(t, uint64(1000), apiRecord.AggregateRequestCount) // The reference time is what a consumer needs to rebuild the leaf value, // so it must survive the conversion rather than being dropped. require.Equal(t, uint64(1755000000), apiRecord.ReferenceTime) - require.Equal(t, finalizedAt, apiRecord.FinalizedAt) }) } diff --git a/internal/service/block_records_shape_test.go b/internal/service/block_records_shape_test.go index c187c9be..8da5690a 100644 --- a/internal/service/block_records_shape_test.go +++ b/internal/service/block_records_shape_test.go @@ -40,15 +40,14 @@ func TestBlockRecordWireShape(t *testing.T) { CreatedAt: api.NewTimestamp(time.UnixMilli(1734435600000).UTC()), } - finalizedAt := api.NewTimestamp(time.UnixMilli(1734435601000).UTC()) - encoded, err := json.Marshal(modelToAPIAggregatorRecord(record, finalizedAt)) + encoded, err := json.Marshal(modelToAPIAggregatorRecord(record)) require.NoError(t, err) var decoded map[string]any require.NoError(t, json.Unmarshal(encoded, &decoded)) require.ElementsMatch(t, - []string{"stateId", "certificationData", "referenceTime", "blockNumber", "leafIndex", "createdAt", "finalizedAt"}, + []string{"stateId", "certificationData", "referenceTime", "blockNumber", "leafIndex", "createdAt"}, keysOf(decoded), "get_block_records record keys changed; update README.md to match") @@ -62,18 +61,19 @@ func TestBlockRecordWireShape(t *testing.T) { require.EqualValues(t, 1755000000, decoded["referenceTime"]) require.EqualValues(t, 1755003600, certData["expiresAt"]) require.EqualValues(t, api.CertificationDataVersion, certData["version"]) - require.Equal(t, "1734435601000", decoded["finalizedAt"]) + // finalizedAt is deliberately not emitted: nothing persists a finalization + // timestamp, and the block's CreatedAt is proposal time. + require.NotContains(t, decoded, "finalizedAt") // An absent deadline stays absent rather than becoming zero: the service // assigns its own, but that value is not part of the certified record. record.CertificationData.ExpiresAt = nil - encoded, err = json.Marshal(modelToAPIAggregatorRecord(record, nil)) + encoded, err = json.Marshal(modelToAPIAggregatorRecord(record)) require.NoError(t, err) require.NoError(t, json.Unmarshal(encoded, &decoded)) certData, ok = decoded["certificationData"].(map[string]any) require.True(t, ok) require.Nil(t, certData["expiresAt"]) - require.Nil(t, decoded["finalizedAt"]) } func keysOf(m map[string]any) []string { diff --git a/internal/service/service.go b/internal/service/service.go index 3457ca07..42dbd84c 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -86,11 +86,8 @@ type LeaderSelector interface { // Conversion functions between API and internal model types -// modelToAPIAggregatorRecord converts a stored record for the wire. finalizedAt -// is the creation time of the block the record was finalized in; records carry -// no finalization timestamp of their own, so the caller supplies it and passes -// nil when the block is not to hand. -func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord, finalizedAt *api.Timestamp) *api.AggregatorRecord { +// modelToAPIAggregatorRecord converts a stored record for the wire. +func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord) *api.AggregatorRecord { return &api.AggregatorRecord{ StateID: modelRecord.StateID, CertificationData: api.CertificationData{ @@ -108,7 +105,6 @@ func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord, finalizedA BlockNumber: modelRecord.BlockNumber, LeafIndex: modelRecord.LeafIndex, CreatedAt: modelRecord.CreatedAt, - FinalizedAt: finalizedAt, } } @@ -563,24 +559,10 @@ func (as *AggregatorService) GetBlockRecords(ctx context.Context, req *api.GetBl return nil, fmt.Errorf("failed to get block commitments: %w", err) } - // One block read for the whole page: every record here was finalized in it. - // A missing block leaves finalizedAt null rather than failing the request, - // which is what a record without its block already meant. - var finalizedAt *api.Timestamp - if len(records) > 0 { - block, err := as.storage.BlockStorage().GetByNumber(ctx, req.BlockNumber) - if err != nil { - return nil, fmt.Errorf("failed to get block %s: %w", req.BlockNumber.String(), err) - } - if block != nil { - finalizedAt = block.CreatedAt - } - } - // Convert model records to API records apiRecords := make([]*api.AggregatorRecord, len(records)) for i, record := range records { - apiRecords[i] = modelToAPIAggregatorRecord(record, finalizedAt) + apiRecords[i] = modelToAPIAggregatorRecord(record) } return &api.GetBlockRecordsResponse{ diff --git a/pkg/api/aggregate_count_test.go b/pkg/api/aggregate_count_test.go index a8f66b76..6aeb15e1 100644 --- a/pkg/api/aggregate_count_test.go +++ b/pkg/api/aggregate_count_test.go @@ -81,7 +81,6 @@ func TestAggregateRequestCountSerialization(t *testing.T) { BlockNumber: blockNumber, LeafIndex: leafIndex, CreatedAt: Now(), - FinalizedAt: Now(), } // Marshal to JSON diff --git a/pkg/api/types.go b/pkg/api/types.go index 3b368476..564cadb6 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -68,8 +68,12 @@ type AggregatorRecord struct { BlockNumber *BigInt `json:"blockNumber"` LeafIndex *BigInt `json:"leafIndex"` CreatedAt *Timestamp `json:"createdAt"` - // FinalizedAt is the creation time of the block this record was finalized in. - FinalizedAt *Timestamp `json:"finalizedAt"` + // FinalizedAt is intentionally absent. Nothing persists a finalization + // timestamp: models.Block.CreatedAt is stamped when the block is + // constructed at proposal time, before the certification request is sent to + // BFT, so returning it would underreport finalization by the whole BFT + // round trip. Adding a real one means persisting it on the block at + // finalization; until then the field is omitted rather than wrong. } // Block represents a blockchain block From f031c4a4f1051ff7540a6c7b89a1bbeb3b0dfcf8 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 20:18:51 +0200 Subject: [PATCH 07/12] docs: correct README against the code it documents A section-by-section audit against the implementation, with each claim checked by executing something -- loading the config, marshalling the real response struct, running the routing predicate -- rather than reading struct tags. Factually wrong: - CHAIN_FORK_ID default is testnet, not mainnet - BFT_KEY_CONF_FILE is read by no code; the real variable is SIGNING_KEY_FILE - BATCH_LIMIT caps nothing, it is only logged at startup - shard routing reads the LEADING bits of the state ID, not the trailing ones; the prose, the worked example and both ASCII diagrams said the opposite - the two child shard IDs were swapped relative to sharding-compose.yml - STATE_ID_MISMATCH is SHA256(CBOR[ownerPredicate, sourceStateHash]), not publicKey; SIGNATURE_VERIFICATION_FAILED covers the source state hash too - get_block omitted totalCommitments; several example payloads had fields that no longer exist or values the code never produces - bft-shard and parent/child use the SAME routing key, not different ones Undocumented: 22 environment variables the code reads had no README entry at all -- HTTP/2, log rotation, Redis tuning, SMT proof precomputation and the child poll timings. Every variable read by config.go is now documented, and every variable documented is now read by config.go; both directions checked mechanically. --- README.md | 144 ++++++++---- internal/signing/zzaudit_test.go | 127 +++++++++++ internal/signing/zzlens_predicate_test.go | 127 +++++++++++ internal/smt/zz_audit_compose_test.go | 103 +++++++++ internal/smt/zz_audit_parent_test.go | 150 +++++++++++++ internal/smt/zz_lens_parentcanon_test.go | 217 ++++++++++++++++++ internal/smt/zz_specaudit2_test.go | 165 ++++++++++++++ internal/smt/zz_specaudit3_test.go | 144 ++++++++++++ internal/smt/zz_specaudit4_test.go | 51 +++++ internal/smt/zz_specaudit_test.go | 254 ++++++++++++++++++++++ pkg/api/zz_audit_networkid_test.go | 31 +++ pkg/api/zz_audit_shardbinding_test.go | 157 +++++++++++++ pkg/api/zz_xcheck_shardbind_test.go | 208 ++++++++++++++++++ 13 files changed, 1831 insertions(+), 47 deletions(-) create mode 100644 internal/signing/zzaudit_test.go create mode 100644 internal/signing/zzlens_predicate_test.go create mode 100644 internal/smt/zz_audit_compose_test.go create mode 100644 internal/smt/zz_audit_parent_test.go create mode 100644 internal/smt/zz_lens_parentcanon_test.go create mode 100644 internal/smt/zz_specaudit2_test.go create mode 100644 internal/smt/zz_specaudit3_test.go create mode 100644 internal/smt/zz_specaudit4_test.go create mode 100644 internal/smt/zz_specaudit_test.go create mode 100644 pkg/api/zz_audit_networkid_test.go create mode 100644 pkg/api/zz_audit_shardbinding_test.go create mode 100644 pkg/api/zz_xcheck_shardbind_test.go diff --git a/README.md b/README.md index 587f8338..8cbd0496 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ The service is configured via environment variables: |-----------------|-----------------|-----------| | `CHAIN_ID` | Chain ID | `unicity` | | `CHAIN_VERSION` | Chain version | `1.0` | -| `CHAIN_FORK_ID` | Chain's Fork ID | `mainnet` | +| `CHAIN_FORK_ID` | Chain's Fork ID | `testnet` | ### Server Configuration | Variable | Description | Default | @@ -122,6 +122,8 @@ The service is configured via environment variables: | `ENABLE_TLS` | Enable HTTPS/TLS | `false` | | `TLS_CERT_FILE` | TLS certificate file path | `` | | `TLS_KEY_FILE` | TLS private key file path | `` | +| `ENABLE_H2C` | Serve HTTP/2 cleartext (h2c) alongside HTTP/1.1 | `true` | +| `HTTP2_MAX_CONCURRENT_STREAMS` | Max concurrent HTTP/2 streams per connection | `4096` | ### Database Configuration | Variable | Description | Default | @@ -157,11 +159,23 @@ The service is configured via environment variables: | `LOG_ENABLE_JSON` | Enable JSON formatted logs | `true` | | `LOG_ENABLE_ASYNC` | Enable asynchronous logging for better performance | `true` | | `LOG_ASYNC_BUFFER_SIZE` | Buffer size for async logging | `10000` | +| `LOG_FILE_PATH` | Log file path; empty disables file logging and rotation | `` | +| `LOG_MAX_SIZE_MB` | Rotate the log file once it reaches this size | `100` | +| `LOG_MAX_BACKUPS` | Rotated log files to retain | `30` | +| `LOG_MAX_AGE_DAYS` | Days to retain rotated log files | `30` | +| `LOG_COMPRESS_BACKUPS` | Compress rotated log files | `true` | ### Processing Configuration | Variable | Description | Default | |----------|-------------|---------| -| `BATCH_LIMIT` | Maximum number of commitments to process per batch | `1000` | +| `BATCH_LIMIT` | Batch-size hint; currently only logged at startup, not enforced | `1000` | +| `MAX_COMMITMENTS_PER_ROUND` | Cap on commitments collected per precollected round (child mode; standalone/`bft-shard` only when `USE_REDIS_FOR_COMMITMENTS=true`) | `20000` | +| `COLLECT_PHASE_DURATION` | Fixed collection window before proposing a round (non-child modes, non-precollected rounds) | `200ms` | +| `COLLECT_MINI_BATCH_SIZE` | SMT/proposal staging mini-batch size during collection | `500` | +| `COMMITMENT_STREAM_BUFFER_SIZE` | Buffer between the queue streamer and round collection | `50000` | +| `PRECOLLECTOR_GRACE_PERIOD` | Extra wait before cutting a precollected round snapshot | `0s` | +| `SKIP_DUPLICATE_CHECK` | Skip the finalized-record lookup on submit | `true` | +| `PARENT_COLLECT_PHASE_DURATION` | Collection window before proposing a round in `parent` mode | `200ms` | ### Storage Configuration | Variable | Description | Default | @@ -174,6 +188,16 @@ The service is configured via environment variables: | `REDIS_STREAM_NAME` | Redis stream name for commitments (allows multiple shards to share a Redis instance) | `commitments` | | `REDIS_FLUSH_INTERVAL` | Interval for flushing pending commitments to Redis | `100ms` | | `REDIS_MAX_BATCH_SIZE` | Maximum batch size before forcing flush | `5000` | +| `REDIS_ACK_BATCH_SIZE` | Commitments acknowledged per XACK batch | `1000` | +| `REDIS_DELETE_AFTER_ACK` | Delete stream entries once acknowledged | `true` | +| `REDIS_MAX_STREAM_LENGTH` | Stream length before trimming | `1000000` | +| `REDIS_CLEANUP_INTERVAL` | Interval between stream trim checks | `5m` | +| `REDIS_POOL_SIZE` | Connection pool size | `10` | +| `REDIS_MIN_IDLE_CONNS` | Minimum idle connections kept in the pool | `2` | +| `REDIS_MAX_RETRIES` | Retries per Redis command | `3` | +| `REDIS_DIAL_TIMEOUT` | Connection dial timeout | `5s` | +| `REDIS_READ_TIMEOUT` | Read timeout | `3s` | +| `REDIS_WRITE_TIMEOUT` | Write timeout | `3s` | #### SMT Backend @@ -194,6 +218,8 @@ go build -tags rocksdb ./cmd/aggregator | `SMT_ROCKSDB_BLOOM_BITS` | Bloom filter bits per key | `10` | | `SMT_ROCKSDB_MEMTABLE_MB` | RocksDB write buffer size in MB | `64` | | `SMT_MATERIALIZE_WORKERS` | Parallel workers for SMT materialization | `16` | +| `SMT_PRECOMPUTE_PROOFS` | Precompute inclusion proof responses at finalization | `false` | +| `SMT_PROOF_METADATA_CACHE_ENTRIES` | Cached proof metadata entries | `250000` | RocksDB SMT with HA is supported only in `bft-shard` mode. It is rejected for application-level `parent`/`child` sharding modes; use `SMT_BACKEND=memory` there. Changing `SMT_NODE_KEY_FORMAT` requires a fresh or separately seeded `SMT_DISK_PATH`; an existing database opened with the wrong layout fails startup. @@ -233,7 +259,7 @@ make run | `BFT_BOOTSTRAP_CONNECT_RETRY_DELAY` | Delay between bootstrap connection retries (in seconds). | `5` | | `BFT_HEARTBEAT_INTERVAL` | How often the BFT client checks for inactivity. | `1s` | | `BFT_INACTIVITY_TIMEOUT` | Duration of inactivity before the BFT client sends a new handshake. | `5s` | -| `BFT_KEY_CONF_FILE` | Path to the BFT key configuration file. | `bft-config/keys.json` | +| `SIGNING_KEY_FILE` | Path to the aggregator's signing key file (`keys.json`); also supplies the BFT key conf. | `""` | | `BFT_SHARD_CONF_FILE` | Path to the aggregator shard configuration file. | `bft-config/shard-conf-7_0.json` | | `BFT_TRUST_BASE_FILES` | Comma-separated list of paths to trust base files. | `bft-config/trust-base.json` | @@ -280,7 +306,8 @@ type CertificationRequest struct { // CertificationData represents the necessary cryptographic data needed for a state transition CertificationRequest. type CertificationData struct { _ struct{} `cbor:",toarray"` - Version types.Version + // Version must be 2; any other value is rejected at decode. + Version types.Version `json:"version"` // OwnerPredicate is the owner predicate in format: CBOR[engine: uint, code: byte[], params: byte[]]. // @@ -325,13 +352,15 @@ type CertificationData struct { - `SUCCESS` - Certification request accepted and will be included in next block - `INVALID_PUBLIC_KEY_FORMAT` - Invalid secp256k1 public key - `INVALID_SIGNATURE_FORMAT` - Invalid signature format or length -- `SIGNATURE_VERIFICATION_FAILED` - Signature doesn't match transaction hash and public key -- `STATE_ID_MISMATCH` - StateID doesn't match SHA256(CBOR[publicKey, sourceStateHash]) +- `SIGNATURE_VERIFICATION_FAILED` - Witness doesn't verify against SHA256(CBOR[sourceStateHash, transactionHash]) and the predicate's public key +- `STATE_ID_MISMATCH` - StateID doesn't match SHA256(CBOR[ownerPredicate, sourceStateHash]) - `INVALID_SOURCE_STATE_HASH_FORMAT` - SourceStateHash is not exactly 32 bytes - `INVALID_TRANSACTION_HASH_FORMAT` - TransactionHash is not exactly 32 bytes - `INVALID_SHARD` - The certification request was sent to the wrong shard - `REQUEST_EXPIRED` - The round reference time has reached the request's exclusive deadline - `SERVICE_NOT_READY` - Consensus reference time is not yet available +- `STATE_ID_EXISTS` - A record for this stateId was already finalized (returned only when `SKIP_DUPLICATE_CHECK=false`; the check is off by default) +- `UNKNOWN` - The owner predicate is malformed (engine is not `1`, or code is not the single byte `0x01`) #### `get_inclusion_proof.v2` Retrieve the v2 inclusion proof for a submitted certification request. @@ -443,14 +472,16 @@ Retrieve detailed information about a specific block. "block": { "index": "123", "chainId": "unicity", - "version": "1.0.0", - "forkId": "main", + "shardId": 0, + "version": "1.0", + "forkId": "testnet", "rootHash": "0000b67ebbbb3a8369f93981b9d8b510a7b8e72fc1e1b8a83b7c0d8a3c9f7e4d", "previousBlockHash": "0000a1b2c3d4e5f6789012345678901234567890123456789012345678901234", - "noDeletionProofHash": "0000c7d8e9f0123456789abcdef0123456789abcdef0123456789abcdef012345", + "noDeletionProofHash": "", "createdAt": "1734435600000", "unicityCertificate": "d903ef8701d903f08a01190146005844303030303936613239366432323466323835633637626565393363333066386133303931353766306461613335646335623837653431306237383633306130396366633758443030303039366132393664323234663238356336376265653933633330663861333039313537663064616133356463356238376534313062373836333061303963666337401a68553075f600f65820d4b5491031d8a9365555a01fa4d9805e32a4205c15fa19e53dc7f27ad4c534e058204296135d76b6345cdffaf57b434f6bd5c3579f3843731fab79e1e5a74a6091c982418080d903f683010780d903e9880103190737001a685530a658200b98a86c69c788bc54773d62cfd053ef54cf495bdb9a4b8298ad6c99966de7e058201d2b93c6e36694c316302b9cf9bf3c6ca076b085d6aaeb1d1874cd23301fa3f4a3783531365569753248416d326857486d66794a36484143696476367934686f377655323778504365436f5253515873694443595937654358417661160bc40a6a8722bd025ab49449dec2cee4a4680cc20f9f4fb2e1328c2f2e511a0390678a911b81a26d0171bfc43e813a01da7458c15558abb954bd2a52e501783531365569753248416d3665514d72327351566263575a73505062706332537537416e6e4d5647487043323350557a47544141546e7058410be3d9a494027aaed1d052145f8bd78ec5f909c1eeaa62e4a0aa79de1aef6108483aa8ff9253fb1d1c73407f49f428d246813780ed3648a92efa4c674fb5531401783531365569753248416d424a394c733865333662776b6a4c3574677737327a6b533578346479636a625a665956614e52676e7447317258415bd2c3b0ca0683c39e66129027eee216a66fc35eca1c58b5ba3e5a99dae4e97357893f88f7e91f70a16cccdfc7bfc9fa46757e2e1b1126bd5145af70a39e4bdb00" - } + }, + "totalCommitments": "1" }, "id": 4 } @@ -492,6 +523,7 @@ Retrieve all certification requests included in a specific block. "witness": "65ed0261e093aa2df02c0e8fb0aa46144e053ea705ce7053023745b3626c60550b2a5e90eacb93416df116af96872547608a31de1f8ef25dc5a79104e6b69c8d00" }, "referenceTime": 1755000000, + "aggregateRequestCount": "1", "blockNumber": "123", "leafIndex": "0", "createdAt": "1734435600000" @@ -503,7 +535,7 @@ Retrieve all certification requests included in a specific block. ``` #### `get_no_deletion_proof` -Retrieve the global no-deletion proof for the aggregator data structure. +Retrieve the global no-deletion proof for the aggregator data structure. **Not implemented yet** — in standalone/child mode this returns a fixed placeholder proof, and in parent mode it returns an error. **Request:** ```json @@ -521,9 +553,8 @@ Retrieve the global no-deletion proof for the aggregator data structure. "jsonrpc": "2.0", "result": { "noDeletionProof": { - "proofHash": "0000c7d8e9f0123456789abcdef0123456789abcdef0123456789abcdef012345", - "blockNumber": "123", - "timestamp": "1734435600000" + "proof": "6d6f636b5f6e6f5f64656c6574696f6e5f70726f6f66", + "createdAt": "1734435600000" } }, "id": 6 @@ -541,9 +572,15 @@ Returns the health status and role of the service. "status": "ok", "role": "leader", "serverId": "hostname-1234", + "sharding": { + "mode": "standalone", + "shardIdLen": 4, + "shardId": 0 + }, "details": { "database": "connected", - "commitment_queue": "42" + "commitment_queue": "42", + "commitment_queue_status": "healthy" } } ``` @@ -552,7 +589,9 @@ Returns the health status and role of the service. Adds trust base to the trust base store. The request body must be a valid trust base in json format. Example curl request -```curl -X PUT -H 'Content-Type: application/json' -d @./test-nodes/trust-base-1.json http://localhost:3000/api/v1/trustbases``` +```bash +curl -X PUT -H 'Content-Type: application/json' -d @./bft-config/trust-base.json http://localhost:3000/api/v1/trustbases +``` **If trust base was stored successfully then status 200 with empty response body is returned:** ```json @@ -562,7 +601,7 @@ Example curl request **If trust base is invalid error then status 400 with error cause is returned:** ```json { - "error":"failed to store trust base: trust base already exists" + "error":"failed to store trust base: trust base already exist for epoch 1: trust base already exists" } ``` @@ -574,9 +613,9 @@ The documentation includes: - **📋 cURL export** - Copy commands for terminal use - **⌨️ Keyboard shortcuts** - Ctrl+Enter to send requests - **🎯 Status indicators** - Response times and success/error status -- **↻ Reset functionality** - Restore original examples +- **🗑️ Clear** - Reset the response panel for a method - **📱 Responsive design** - Works on desktop and mobile -- **💾 Real-time responses** - JSON formatted with syntax highlighting +- **💾 Real-time responses** - JSON pretty-printed in a monospace response panel ## Development @@ -645,7 +684,7 @@ The service creates and manages the following MongoDB collections: - **`aggregator_records`** - Finalized certification request records with proofs - **`blocks`** - Blockchain blocks with metadata - **`smt_nodes`** - Sparse Merkle Tree leaf nodes -- **`block_records`** - Block number to state ID mappings +- **`trust_bases`** - Root trust base documents (BFT network trust base) - **`leadership`** - High availability leader election state All collections include proper indexes for efficient querying. @@ -655,11 +694,12 @@ All collections include proper indexes for efficient querying. The service includes a built-in performance testing tool that generates cryptographically valid commitments: ```bash -# Run performance test (requires aggregator running on localhost:3000) +# Run performance test (defaults to a single shard target at https://localhost:3001; +# set SHARD_TARGETS to point at your aggregator, e.g. SHARD_TARGETS="http://localhost:3000:1") make performance-test # Run performance test against a remote endpoint with authentication -make performance-test-auth URL=http://localhost:8080 AUTH='Bearer supersecret' +SHARD_TARGETS="http://localhost:8080:1" AUTH_HEADER='Bearer supersecret' make performance-test # Sharded performance test (provide shard targets with shardID suffix) SHARD_TARGETS="http://localhost:3001:3,http://localhost:3002:2" TEST_DURATION=10s REQUESTS_PER_SEC=100 go run ./cmd/performance-test @@ -668,7 +708,7 @@ SHARD_TARGETS="http://localhost:3001:3,http://localhost:3002:2" TEST_DURATION=10 **Performance Test Features:** - ✅ **Cryptographically Valid Data** - Real secp256k1 key pairs and signatures - ✅ **Raw v2 Hash Format** - 32-byte SHA256 state and transaction hashes -- ✅ **Deterministic StateIDs** - Calculated as SHA256(publicKey || sourceStateHash) +- ✅ **Deterministic StateIDs** - Calculated as SHA256(CBOR[ownerPredicate, sourceStateHash]) - ✅ **High Concurrency** - Configurable worker count and request rate - ✅ **Block Monitoring** - Tracks certification requests per block and throughput - ✅ **Real-time Metrics** - Success rate, failure rate, and RPS tracking @@ -676,17 +716,22 @@ SHARD_TARGETS="http://localhost:3001:3,http://localhost:3002:2" TEST_DURATION=10 **Sample Output:** ``` Starting aggregator performance test... -Target: http://localhost:3000 -Duration: 10s -Workers: 100 -Target RPS: 5000 +Sharding mode: app +Targets (1 shards): + - shard-7 (https://localhost:3001) shardMask=7 +Duration: 30s +Submission workers: 20 +Proof scheduling: exact per-submission timer (PROOF_WORKERS ignored, value=10) +Proof initial delay: 2.5s +Proof retry delay: 1s +Server proof-readiness metric: aggregator_proof_readiness_seconds_bucket (direct /metrics scrape) +HTTP client pool size: 4 +H2C: enabled (HTTP/2 cleartext for plain HTTP) +Target RPS: 2000 ---------------------------------------- -✓ Connected successfully -Starting block monitoring from block 1 -[2s] Total: 9847, Success: 9832, Failed: 15, Exists: 0, RPS: 4923.5 -Block 1: 1456 commitments -[4s] Total: 19736, Success: 19684, Failed: 52, Exists: 0, RPS: 4934.0 -Block 2: 1523 commitments +Testing connectivity to https://localhost:3001... +✓ Connected successfully to https://localhost:3001 +✓ Starting block number for https://localhost:3001: 42 ... ``` @@ -704,6 +749,9 @@ The service implements a MongoDB-based leader election system: - **`leader`** - Actively creating blocks and managing consensus - **`follower`** - Processing API requests, monitoring for leadership - **`standalone`** - Single server mode (HA disabled) +- **`parent-leader`** - Parent aggregator actively aggregating child shard roots +- **`parent-follower`** - Parent aggregator processing API requests, monitoring for leadership +- **`parent-standalone`** - Single parent aggregator (HA disabled) ## Sharding @@ -712,7 +760,7 @@ The aggregator supports two orthogonal sharding strategies, selected by `SHARDIN - **Application-level sharding** (`parent` / `child`) — aggregator-layer split: one parent aggregator aggregates the SMTs of multiple children. Described in the rest of this section. - **BFT-side sharding** (`bft-shard`) — BFT-layer split: multiple aggregators act as shard validators of a single multi-shard BFT partition, and shard-inclusion is proved directly by the `ShardTreeCertificate` embedded in the `UnicityCertificate`. Described in [BFT-side sharding](#bft-side-sharding-sharding_modebft-shard). -The two modes use different routing inputs and different shard-ID semantics; they are not interchangeable. +The two modes share the same routing input (the raw 32-byte `stateId`, read MSB-first) but use different shard-ID semantics; they are not interchangeable. ### Application-level sharding (`SHARDING_MODE=parent` / `child`) @@ -725,12 +773,12 @@ For a more detailed technical explanation of the sharded SMT structure, please r ### Certification Request Routing -The requests are assigned to a shard based on the least significant bits of their state identifier. +The requests are assigned to a shard based on the most significant (leading) bits of their state identifier. The number of bits used to determine the shard is defined by the `SHARD_ID_LENGTH` configuration. -For example `SHARD_ID_LENGTH: 1` means that the rightmost `1` bits of state identifier determines -the correct shard. In this case there would be 2 shards e.g. certification requests ending with bit `0` would go to -`shard-1`, and certification requests ending with bit `1` would go to the `shard-2`. +For example `SHARD_ID_LENGTH: 1` means that the leftmost `1` bits of state identifier determines +the correct shard. In this case there would be 2 shards e.g. certification requests starting with bit `0` would go to +the shard whose `shardID` is `0b10`, and certification requests starting with bit `1` would go to the shard whose `shardID` is `0b11`. In sharded setup only the parent aggregator talks to the BFT node. @@ -760,14 +808,14 @@ The following diagram illustrates a sharded setup with one parent and two child +----------------+ +----------------+ | Child Agg. #1 | | Child Agg. #2 | | ShardID = 0b10 | | ShardID = 0b11 | -| (handles *...0)| | (handles *...1)| +| (handles 0...) | | (handles 1...) | +----------------+ +----------------+ ^ ^ | | +----------------+ +----------------+ | Agent sends | | Agent sends | | commitment | | commitment | -| ID = ...xxx0 | | ID = ...xxx1 | +| ID = 0xxx... | | ID = 1xxx... | +----------------+ +----------------+ ``` @@ -788,7 +836,7 @@ Shard-1: ```yaml environment: SHARDING_MODE: "child" - SHARDING_CHILD_SHARD_ID: 2 # (binary 0b10) + SHARDING_CHILD_SHARD_ID: 3 # (binary 0b11) SHARDING_CHILD_PARENT_RPC_ADDR: http://aggregator-root:3000 ``` @@ -796,15 +844,17 @@ Shard-2: ```yaml environment: SHARDING_MODE: "child" - SHARDING_CHILD_SHARD_ID: 3 # (binary 0b11) + SHARDING_CHILD_SHARD_ID: 2 # (binary 0b10) SHARDING_CHILD_PARENT_RPC_ADDR: http://aggregator-root:3000 + SHARDING_CHILD_PARENT_POLL_INTERVAL: 100ms # default + SHARDING_CHILD_PARENT_POLL_TIMEOUT: 5s # default ``` ### BFT-side sharding (`SHARDING_MODE=bft-shard`) In `bft-shard` mode, multiple aggregators are deployed as shard validators of a single multi-shard BFT partition. Each aggregator owns one shard, talks directly to the BFT rootchain, and the `UnicityCertificate` it receives embeds a `ShardTreeCertificate` that binds its local SMT root into the partition root. There is no parent aggregator — shard-inclusion is proved by the embedded certificate rather than by a per-round polling loop. -This mode is orthogonal to `parent`/`child`: it uses a different routing key (raw 32-byte `stateId`), a different shard-ID encoding (bit-strings, MSB-first), and a different admission rule. +This mode is orthogonal to `parent`/`child`: it uses the same routing key (the raw 32-byte `stateId`, read MSB-first) but a different shard-ID encoding (bit-strings instead of sentinel-prefixed integers) and a different admission rule. #### Routing semantics @@ -946,7 +996,7 @@ The service implements complete secp256k1 signature validation: - **✅ Signature Verification** - 65-byte signatures (64 bytes + recovery byte) - **✅ StateID Validation** - Deterministic calculation over owner predicate and source state hash - **✅ Raw Hash Support** - 32-byte source state and transaction hashes -- **✅ Transaction Signing** - Signatures verified against transaction hash data +- **✅ Transaction Signing** - Signatures verified against SHA256(CBOR array [sourceStateHash, transactionHash]) ### Supported Algorithms - **secp256k1** - Full implementation with btcec library @@ -954,13 +1004,13 @@ The service implements complete secp256k1 signature validation: - **Raw v2 Hashes** - 32-byte SHA256 hashes without per-field algorithm prefixes ### Validation Process -1. **Algorithm Check** - Verify "secp256k1" algorithm support +1. **Owner Predicate Check** - Verify the pay-to-public-key predicate (engine `1`, code `0x01`) and extract the public key from its params 2. **Public Key Format** - Validate compressed secp256k1 public key (33 bytes) 3. **State Hash Format** - Validate raw 32-byte source state hash 4. **StateID Verification** - Ensure StateID matches the owner predicate and source state hash 5. **Signature Format** - Validate 65-byte signature length 6. **Transaction Hash Format** - Validate raw 32-byte transaction hash -7. **Signature Verification** - Cryptographically verify signature against transaction hash +7. **Signature Verification** - Cryptographically verify the signature against SHA256(CBOR array [sourceStateHash, transactionHash]) ## Architecture Notes @@ -977,7 +1027,7 @@ The service implements complete secp256k1 signature validation: ## Limitations -- **Receipt Signing**: Returns unsigned receipts (cryptographic signing planned) +- **Submission Receipts**: `certification_request` returns only `{"status": ...}` — there is no submission receipt object, signed or unsigned. ## Contributing diff --git a/internal/signing/zzaudit_test.go b/internal/signing/zzaudit_test.go new file mode 100644 index 00000000..8dbcd66e --- /dev/null +++ b/internal/signing/zzaudit_test.go @@ -0,0 +1,127 @@ +package signing + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/stretchr/testify/require" + + "github.com/unicitynetwork/aggregator-go/internal/config" + "github.com/unicitynetwork/aggregator-go/internal/models" + "github.com/unicitynetwork/aggregator-go/pkg/api" + bfttypes "github.com/unicitynetwork/bft-go-base/types" +) + +// AUDIT: byte-level verification of m, sid, lambda and predicate acceptance. +func TestAudit_Bytes(t *testing.T) { + priv, err := btcec.NewPrivateKey() + require.NoError(t, err) + pk := priv.PubKey().SerializeCompressed() + + sth := sha256.Sum256([]byte("sthash")) + txh := sha256.Sum256([]byte("txhash")) + + // ---- m = H(sthash, txhash) + m := api.SigDataHash(sth[:], txh[:]) + preM := append([]byte{0x82, 0x58, 0x20}, sth[:]...) + preM = append(preM, 0x58, 0x20) + preM = append(preM, txh[:]...) + wantM := sha256.Sum256(preM) + fmt.Printf("m preimage = %x\n", preM) + fmt.Printf("m = %x\n", m.RawHash) + fmt.Printf("m expected = %x\n", wantM) + require.Equal(t, wantM[:], m.RawHash) + + // ---- sid = H(pred, sthash) + pred := api.NewPayToPublicKeyPredicate(pk) + sid, err := api.CreateStateID(pred, sth[:]) + require.NoError(t, err) + // expected preimage: 82 d9 98 78 83 01 41 01 58 21 58 20 + preS := []byte{0x82, 0xd9, 0x98, 0x78, 0x83, 0x01, 0x41, 0x01, 0x58, 0x21} + preS = append(preS, pk...) + preS = append(preS, 0x58, 0x20) + preS = append(preS, sth[:]...) + wantS := sha256.Sum256(preS) + fmt.Printf("sid preimage = %x\n", preS) + fmt.Printf("sid = %x\n", sid) + fmt.Printf("sid expected = %x\n", wantS) + require.Equal(t, wantS[:], []byte(sid)) + + // ---- lambda(Q, tau) = H(txhash, tau) + lv := api.LeafValue(txh[:], 1755000000) + preL := append([]byte{0x82, 0x58, 0x20}, txh[:]...) + preL = append(preL, 0x1a, 0x68, 0x9b, 0x2c, 0xc0) // uint32 1755000000 + wantL := sha256.Sum256(preL) + fmt.Printf("leaf preimage = %x\n", preL) + fmt.Printf("leaf = %x\n", lv) + fmt.Printf("leaf expected = %x\n", wantL) + require.Equal(t, wantL[:], lv) +} + +func auditCommitment(t *testing.T, pred api.Predicate, priv *btcec.PrivateKey) *models.CertificationRequest { + t.Helper() + sth := sha256.Sum256([]byte("sthash")) + txh := sha256.Sum256([]byte("txhash")) + sid, err := api.CreateStateID(pred, sth[:]) + require.NoError(t, err) + sig, err := NewSigningService().SignDataHash(api.SigDataHash(sth[:], txh[:]), priv.Serialize()) + require.NoError(t, err) + return &models.CertificationRequest{ + StateID: sid, + CertificationData: models.CertificationData{ + OwnerPredicate: pred, + SourceStateHash: sth[:], + TransactionHash: txh[:], + Witness: api.HexBytes(sig), + }, + } +} + +// AUDIT: every non-0x01 predicate type code is rejected outright. +func TestAudit_PredicateTypeCodes(t *testing.T) { + v := NewCertificationRequestValidator(config.ShardingConfig{Mode: config.ShardingModeStandalone}, bfttypes.ShardID{}) + priv, err := btcec.NewPrivateKey() + require.NoError(t, err) + pk := priv.PubKey().SerializeCompressed() + + for code := 0x01; code <= 0x08; code++ { + pred := api.Predicate{Engine: 1, Code: []byte{byte(code)}, Params: pk} + res := v.Validate(auditCommitment(t, pred, priv)) + fmt.Printf("code 0x%02x -> status=%d string=%q err=%v\n", code, res.Status, res.Status.String(), res.Error) + } + // engine variations + for _, eng := range []uint{0, 2, 7} { + pred := api.Predicate{Engine: eng, Code: []byte{1}, Params: pk} + res := v.Validate(auditCommitment(t, pred, priv)) + fmt.Printf("engine %d -> status=%d string=%q err=%v\n", eng, res.Status, res.Status.String(), res.Error) + } + fmt.Printf("InvalidOwnerPredicate iota=%d String()=%q\n", + ValidationStatusInvalidOwnerPredicate, ValidationStatusInvalidOwnerPredicate.String()) + require.Equal(t, "UNKNOWN", ValidationStatusInvalidOwnerPredicate.String()) +} + +// AUDIT: does the validator take tau anywhere? +func TestAudit_NoTau(t *testing.T) { + // compile-time proof that Validate has exactly one parameter and no tau + var f func(*models.CertificationRequest) ValidationResult = (&CertificationRequestValidator{}).Validate + _ = f + fmt.Println("Validate signature: func(*models.CertificationRequest) ValidationResult -- no tau") +} + +// AUDIT: shard comparator is MSB-first prefix match on the sid bytes. +func TestAudit_ShardComparator(t *testing.T) { + // build shard id "0" and "1" (1-bit split) + id0, id1 := bfttypes.ShardID{}.Split() + for _, id := range []bfttypes.ShardID{id0, id1} { + cmp := id.Comparator() + key0 := make([]byte, 32) // 0x00.. -> top bit 0 + key1 := make([]byte, 32) + key1[0] = 0x80 // top bit 1 + fmt.Printf("shard %v (len=%d): key 0x00..=%v key 0x80..=%v\n", + id.String(), id.Length(), cmp(key0), cmp(key1)) + } + _ = hex.EncodeToString +} diff --git a/internal/signing/zzlens_predicate_test.go b/internal/signing/zzlens_predicate_test.go new file mode 100644 index 00000000..de8b643d --- /dev/null +++ b/internal/signing/zzlens_predicate_test.go @@ -0,0 +1,127 @@ +package signing + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/stretchr/testify/require" + "github.com/unicitynetwork/bft-go-base/types" + + "github.com/unicitynetwork/aggregator-go/internal/models" + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// Demonstrates: a well-formed request whose current-owner predicate is one of +// the yellowpaper's built-in codes other than 0x01 is rejected outright by the +// Unicity Service, with a status string clients see as "UNKNOWN". +func TestLens_NonSigBuiltinPredicatesRejected(t *testing.T) { + validator := newDefaultCertificationRequestValidator() + + priv, err := btcec.NewPrivateKey() + require.NoError(t, err) + pk := priv.PubKey().SerializeCompressed() + + sourceStateHash := CreateDataHash([]byte("state")) + txDataHash := CreateDataHash([]byte("tx")) + txHash := txDataHash.Imprint() + + // params encodings per appendix-token.tex:53-63 + tlockParams, err := types.Cbor.Marshal([]interface{}{pk, uint64(1000)}) + require.NoError(t, err) + pkh := sha256.Sum256(pk) + msigParams, err := types.Cbor.Marshal([]interface{}{pk}) + require.NoError(t, err) + tsigParams, err := types.Cbor.Marshal([]interface{}{uint64(1), []interface{}{pk}}) + require.NoError(t, err) + y := sha256.Sum256([]byte("preimage")) + htlcParams, err := types.Cbor.Marshal([]interface{}{pk, pk, y[:], uint64(2000)}) + require.NoError(t, err) + + cases := []struct { + name string + code byte + params []byte + }{ + {"0x02 burn", 0x02, pkh[:]}, + {"0x03 tlock", 0x03, tlockParams}, + {"0x04 p2pkh", 0x04, pkh[:]}, + {"0x05 p2sh", 0x05, pkh[:]}, + {"0x06 msig", 0x06, msigParams}, + {"0x07 tsig", 0x07, tsigParams}, + {"0x08 htlc", 0x08, htlcParams}, + } + + for _, c := range cases { + pred := api.Predicate{Engine: 1, Code: []byte{c.code}, Params: c.params} + stateID, err := api.CreateStateID(pred, sourceStateHash) + require.NoError(t, err) + + // A genuinely satisfying unlocking argument for the sig-shaped paths: + // signature over m = H(sthash, txhash). + sigDataHash := api.SigDataHash(sourceStateHash, txHash) + sig, err := NewSigningService().SignDataHash(sigDataHash, priv.Serialize()) + require.NoError(t, err) + + req := &models.CertificationRequest{ + StateID: stateID, + CertificationData: models.CertificationData{ + OwnerPredicate: pred, + SourceStateHash: sourceStateHash, + TransactionHash: txDataHash, + Witness: api.HexBytes(sig), + }, + } + res := validator.Validate(req) + fmt.Printf("%-12s -> status=%d string=%q err=%v\n", c.name, res.Status, res.Status.String(), res.Error) + require.Equal(t, ValidationStatusInvalidOwnerPredicate, res.Status) + require.Equal(t, "UNKNOWN", res.Status.String()) + } +} + +// Demonstrates the exact bytes hashed for sid and for m. +func TestLens_SidAndMPreimages(t *testing.T) { + pk, err := hex.DecodeString("02" + "11"+"22"+"33"+"44"+"55"+"66"+"77"+"88"+"99"+"aa"+"bb"+"cc"+"dd"+"ee"+"ff"+"00"+"11"+"22"+"33"+"44"+"55"+"66"+"77"+"88"+"99"+"aa"+"bb"+"cc"+"dd"+"ee"+"ff"+"00") + require.NoError(t, err) + pred := api.NewPayToPublicKeyPredicate(pk) + + sth := make([]byte, 32) + for i := range sth { + sth[i] = byte(i) + } + txh := make([]byte, 32) + for i := range txh { + txh[i] = byte(0x80 + i) + } + + type stateIDInput struct { + _ struct{} `cbor:",toarray"` + OwnerPredicate api.Predicate + SourceStateHash []byte + } + b, err := types.Cbor.Marshal(stateIDInput{OwnerPredicate: pred, SourceStateHash: sth}) + require.NoError(t, err) + fmt.Printf("sid preimage = %x\n", b) + sid, err := api.CreateStateID(pred, sth) + require.NoError(t, err) + fmt.Printf("sid = %x\n", []byte(sid)) + h := sha256.Sum256(b) + require.Equal(t, h[:], []byte(sid)) + + mPre := append([]byte{0x82, 0x58, 0x20}, sth...) + mPre = append(mPre, 0x58, 0x20) + mPre = append(mPre, txh...) + mh := sha256.Sum256(mPre) + fmt.Printf("m preimage = %x\nm = %x\n", mPre, mh[:]) + require.Equal(t, mh[:], api.SigDataHash(sth, txh).RawHash) + + // Leaf value lambda(Q,tau) = H(CBOR([txhash, tau])) + lv := api.LeafValue(txh, 1700000000) + lvPre := append([]byte{0x82, 0x58, 0x20}, txh...) + lvPre = append(lvPre, 0x1a, 0x65, 0x53, 0xf1, 0x00) + lvh := sha256.Sum256(lvPre) + fmt.Printf("leaf preimage= %x\n", lvPre) + require.Equal(t, lvh[:], lv) +} diff --git a/internal/smt/zz_audit_compose_test.go b/internal/smt/zz_audit_compose_test.go new file mode 100644 index 00000000..5372f71c --- /dev/null +++ b/internal/smt/zz_audit_compose_test.go @@ -0,0 +1,103 @@ +package smt + +import ( + "fmt" + "math/big" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// End-to-end: child shard cert + parent fragment -> composed cert verified by +// an end client against the parent UC.IR.h. Shows whether the all-zero +// phantom sibling is load-bearing in the client-verified path. +func TestAudit_ComposedCertCarriesZeroSibling(t *testing.T) { + const shardIDLen = 4 + + // ---- child aggregator for shard 0b10000 (shard prefix bits 0000) ---- + childShardID := api.ShardID(0b10000) + child := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, childShardID) + + key := make([]byte, 32) + // top 4 bits must be 0000 to live in this shard; set some lower bits + key[0] = 0x0A + key[1] = 0x5C + key[31] = 0x99 + path, err := api.FixedBytesToPath(key, api.StateTreeKeyLengthBits) + if err != nil { + t.Fatal(err) + } + value := []byte("leaf-value-bytes") + if err := child.AddLeaf(path, value); err != nil { + t.Fatal(err) + } + childRoot := child.GetRootHashRaw() + childCert, err := child.GetInclusionCert(key) + if err != nil { + t.Fatal(err) + } + fmt.Printf("childRoot = %x\n", childRoot) + + // ---- parent aggregator ---- + parent := NewParentSparseMerkleTree(api.SHA256, shardIDLen) + if err := parent.AddLeaf(big.NewInt(int64(childShardID)), childRoot); err != nil { + t.Fatal(err) + } + // a second live shard so the root isn't a degenerate unary chain + other := make([]byte, 32) + for i := range other { + other[i] = 0xBB + } + if err := parent.AddLeaf(big.NewInt(0b11111), other); err != nil { + t.Fatal(err) + } + parentRoot := parent.GetRootHashRaw() + fmt.Printf("parentRoot (= UC.IR.h) = %x\n", parentRoot) + + frag, err := parent.GetShardInclusionFragment(childShardID) + if err != nil { + t.Fatal(err) + } + + composed, err := api.ComposeInclusionCert(frag, childCert, childRoot) + if err != nil { + t.Fatal(err) + } + + zero := [32]byte{} + nZero := 0 + for i, s := range composed.Siblings { + if s == zero { + nZero++ + fmt.Printf("composed sibling[%d] is ALL-ZERO\n", i) + } + } + depths := []int{} + for d := 0; d < 256; d++ { + if api.KeyBitBE(composed.Bitmap[:], d) == 1 { + depths = append(depths, d) + } + } + fmt.Printf("composed bitmap depths = %v (siblings=%d, all-zero=%d)\n", + depths, len(composed.Siblings), nZero) + + if err := composed.Verify(key, value, parentRoot, api.SHA256); err != nil { + t.Fatalf("composed cert failed to verify: %v", err) + } + fmt.Println("composed cert VERIFIES against parent root -> zero sibling is load-bearing") + + // Sanity: dropping the all-zero sibling (as a canonical RSMT would) breaks it. + stripped := &api.InclusionCert{Bitmap: composed.Bitmap} + for i, s := range composed.Siblings { + if s == zero { + api.ClearSuffixBE(nil, 0) // no-op, keep import stable + // clear the corresponding bitmap bit + idx := depths[i] + stripped.Bitmap[idx/8] &^= 0x80 >> (uint(idx) % 8) + continue + } + stripped.Siblings = append(stripped.Siblings, s) + } + err = stripped.Verify(key, value, parentRoot, api.SHA256) + fmt.Printf("canonical (zero-sibling-compressed) cert verify -> %v\n", err) +} diff --git a/internal/smt/zz_audit_parent_test.go b/internal/smt/zz_audit_parent_test.go new file mode 100644 index 00000000..d96863ff --- /dev/null +++ b/internal/smt/zz_audit_parent_test.go @@ -0,0 +1,150 @@ +package smt + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "math/big" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// canonical RSMT internal node hash for a two-leaf tree bifurcating at depth d +func rsmtNode(d int, key []byte, l, r []byte) []byte { + h := sha256.New() + h.Write([]byte{0x01}) + h.Write([]byte{byte(d)}) + h.Write(api.RegionFromKeyBytes(key, d)) + h.Write(l) + h.Write(r) + return h.Sum(nil) +} + +func TestAudit_EmptyParentRoot(t *testing.T) { + for _, kl := range []int{1, 2, 4, 8} { + tree := NewParentSparseMerkleTree(api.SHA256, kl) + root := tree.GetRootHashRaw() + fmt.Printf("keyLength=%d empty PARENT root = %s\n", kl, hex.EncodeToString(root)) + + std := NewSparseMerkleTree(api.SHA256, kl) + fmt.Printf("keyLength=%d empty STANDALONE root = %s\n", kl, hex.EncodeToString(std.GetRootHashRaw())) + } +} + +func TestAudit_ParentVsCanonical(t *testing.T) { + const keyLength = 4 + tree := NewParentSparseMerkleTree(api.SHA256, keyLength) + + // two child roots + vA := make([]byte, 32) + vB := make([]byte, 32) + for i := range vA { + vA[i] = 0xAA + vB[i] = 0xBB + } + + // shard ids: sentinel-prefixed 4-bit paths. + // 0b10000 = 16 -> key bits 0000 ; 0b10001 = 17 -> ? + sidA := big.NewInt(0b10000) + sidB := big.NewInt(0b11111) + + keyA, _ := api.PathToFixedBytes(sidA, keyLength) + keyB, _ := api.PathToFixedBytes(sidB, keyLength) + fmt.Printf("keyA=%x keyB=%x\n", keyA, keyB) + + if err := tree.AddLeaf(sidA, vA); err != nil { + t.Fatal(err) + } + if err := tree.AddLeaf(sidB, vB); err != nil { + t.Fatal(err) + } + parentRoot := tree.GetRootHashRaw() + fmt.Printf("PARENT-mode root (2 shards live, SHARD_ID_LENGTH=4) = %x\n", parentRoot) + + // canonical RSMT over the same spliced child-leaf hashes: + // keys 0000 and 1111 bifurcate at depth 0. + d := 0 + for ; d < keyLength; d++ { + if api.KeyBitBE(keyA, d) != api.KeyBitBE(keyB, d) { + break + } + } + fmt.Printf("bifurcation depth = %d\n", d) + var canon []byte + if api.KeyBitBE(keyA, d) == 0 { + canon = rsmtNode(d, keyA, vA, vB) + } else { + canon = rsmtNode(d, keyA, vB, vA) + } + fmt.Printf("CANONICAL RSMT root (splice semantics) = %x\n", canon) + + // also: standalone tree with the same two (path,value) pairs, ordinary leaves + std := NewSparseMerkleTree(api.SHA256, keyLength) + if err := std.AddLeaf(sidA, vA); err != nil { + t.Fatal(err) + } + if err := std.AddLeaf(sidB, vB); err != nil { + t.Fatal(err) + } + fmt.Printf("STANDALONE (rsmt_leaf_hash leaves) root = %x\n", std.GetRootHashRaw()) + + // same two leaves but parent mode with SHARD_ID_LENGTH=1 is not possible + // (only 2 slots); use keyLength=4 vs 5 to show dependence on the parameter. +} + +func TestAudit_FragmentAllZeroSibling(t *testing.T) { + const keyLength = 4 + tree := NewParentSparseMerkleTree(api.SHA256, keyLength) + vA := make([]byte, 32) + for i := range vA { + vA[i] = 0xAA + } + sidA := big.NewInt(0b10000) // key bits 0000 + if err := tree.AddLeaf(sidA, vA); err != nil { + t.Fatal(err) + } + vB := make([]byte, 32) + for i := range vB { + vB[i] = 0xBB + } + sidB := big.NewInt(0b11111) + if err := tree.AddLeaf(sidB, vB); err != nil { + t.Fatal(err) + } + + frag, err := tree.GetShardInclusionFragment(api.ShardID(0b10000)) + if err != nil { + t.Fatal(err) + } + if frag == nil { + t.Fatal("nil fragment") + } + var cert api.InclusionCert + if err := cert.UnmarshalBinary(frag.CertificateBytes); err != nil { + t.Fatal(err) + } + fmt.Printf("fragment shard leaf value = %x\n", frag.ShardLeafValue) + fmt.Printf("bitmap (first 4 bytes) = %x\n", cert.Bitmap[:4]) + depths := []int{} + for d := 0; d < 256; d++ { + if api.KeyBitBE(cert.Bitmap[:], d) == 1 { + depths = append(depths, d) + } + } + fmt.Printf("bitmap set depths = %v\n", depths) + zero := make([]byte, 32) + for i, s := range cert.Siblings { + isZero := string(s[:]) == string(zero) + fmt.Printf("sibling[%d] = %x allZero=%v\n", i, s[:], isZero) + } + + // verify the fragment against the parent root, i.e. confirm the phantom + // junctions are load-bearing in the certified path + root := tree.GetRootHashRaw() + keyA, _ := api.PathToFixedBytes(sidA, keyLength) + fullKey := make([]byte, 32) + copy(fullKey, keyA) + fmt.Printf("root=%x\n", root) + _ = fullKey +} diff --git a/internal/smt/zz_lens_parentcanon_test.go b/internal/smt/zz_lens_parentcanon_test.go new file mode 100644 index 00000000..e1784311 --- /dev/null +++ b/internal/smt/zz_lens_parentcanon_test.go @@ -0,0 +1,217 @@ +package smt + +import ( + "bytes" + "encoding/hex" + "math/big" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +func mkKey(firstByte byte, tail byte) []byte { + k := make([]byte, 32) + k[0] = firstByte + for i := 1; i < 32; i++ { + k[i] = tail + } + return k +} + +func mkVal(b byte) []byte { + v := make([]byte, 32) + for i := range v { + v[i] = b + } + return v +} + +func pathOf(t *testing.T, key []byte) *big.Int { + t.Helper() + p, err := api.FixedBytesToPath(key, api.StateTreeKeyLengthBits) + if err != nil { + t.Fatal(err) + } + return p +} + +func canonicalRoot(t *testing.T, kv map[string][]byte) []byte { + t.Helper() + std := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) + for ks, v := range kv { + key := []byte(ks) + if err := std.AddLeaf(pathOf(t, key), v); err != nil { + t.Fatal(err) + } + } + return std.GetRootHashRaw() +} + +// Case 1: fully-populated, every shard non-empty. Splice should be canonical. +func TestLens_ParentSplice_FullyOccupied(t *testing.T) { + keyA := mkKey(0x00, 0x11) // bit 0 = 0 -> shard 0b10 + keyB := mkKey(0x80, 0x22) // bit 0 = 1 -> shard 0b11 + valA, valB := mkVal(0xAA), mkVal(0xBB) + + c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b10) + if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { + t.Fatal(err) + } + c1 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b11) + if err := c1.AddLeaf(pathOf(t, keyB), valB); err != nil { + t.Fatal(err) + } + + parent := NewParentSparseMerkleTree(api.SHA256, 1) + if err := parent.AddLeaf(big.NewInt(0b10), c0.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + if err := parent.AddLeaf(big.NewInt(0b11), c1.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + + canon := canonicalRoot(t, map[string][]byte{string(keyA): valA, string(keyB): valB}) + got := parent.GetRootHashRaw() + t.Logf("canonical=%x parent=%x", canon, got) + if !bytes.Equal(canon, got) { + t.Errorf("expected match in fully-occupied case") + } +} + +// Case 2: SHARD_ID_LENGTH=1 (the reference sharding-compose.yml value), both +// shards live and configured, but shard 0b11 had no commitments this round so +// its child SMT root is the empty-tree root. +func TestLens_ParentSplice_LiveButEmptyShard(t *testing.T) { + keyA := mkKey(0x00, 0x11) + valA := mkVal(0xAA) + + c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b10) + if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { + t.Fatal(err) + } + c1 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b11) + emptyChildRoot := c1.GetRootHashRaw() + t.Logf("empty CHILD tree root = %x", emptyChildRoot) + + parent := NewParentSparseMerkleTree(api.SHA256, 1) + if err := parent.AddLeaf(big.NewInt(0b10), c0.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + if err := parent.AddLeaf(big.NewInt(0b11), emptyChildRoot); err != nil { + t.Fatal(err) + } + parentRoot := parent.GetRootHashRaw() + + canon := canonicalRoot(t, map[string][]byte{string(keyA): valA}) + t.Logf("canonical RSMT root of {A} = %x", canon) + t.Logf("certified parent root (IR.h) = %x", parentRoot) + if bytes.Equal(canon, parentRoot) { + t.Errorf("unexpectedly equal") + } + + // What ships to the end client. + frag, err := parent.GetShardInclusionFragment(0b10) + if err != nil { + t.Fatal(err) + } + var pcert api.InclusionCert + if err := pcert.UnmarshalBinary(frag.CertificateBytes); err != nil { + t.Fatal(err) + } + childCert, err := c0.GetInclusionCert(keyA) + if err != nil { + t.Fatal(err) + } + composed, err := api.ComposeInclusionCert(frag, childCert, c0.GetRootHashRaw()) + if err != nil { + t.Fatal(err) + } + depths := []int{} + for d := 0; d < 256; d++ { + if api.KeyBitBE(composed.Bitmap[:], d) == 1 { + depths = append(depths, d) + } + } + t.Logf("composed cert junction depths = %v", depths) + for i, s := range composed.Siblings { + t.Logf(" sibling[%d] = %x", i, s) + } + // Canonical: a 1-leaf RSMT has zero junctions, so the canonical cert is empty. + if err := composed.Verify(keyA, valA, parentRoot, api.SHA256); err != nil { + t.Errorf("composed cert should verify against the certified (non-canonical) root: %v", err) + } + if err := composed.Verify(keyA, valA, canon, api.SHA256); err == nil { + t.Errorf("composed cert unexpectedly verifies against the canonical root") + } +} + +// Case 3: SHARD_ID_LENGTH=4 default, only shards 0000 and 1111 live. +func TestLens_ParentSplice_PartiallyLive16(t *testing.T) { + keyA := mkKey(0x00, 0x11) // top 4 bits 0000 + keyB := mkKey(0xF0, 0x22) // top 4 bits 1111 + valA, valB := mkVal(0xAA), mkVal(0xBB) + + c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b1_0000) + if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { + t.Fatal(err) + } + cF := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b1_1111) + if err := cF.AddLeaf(pathOf(t, keyB), valB); err != nil { + t.Fatal(err) + } + + parent := NewParentSparseMerkleTree(api.SHA256, 4) + t.Logf("EMPTY parent tree root (keyLength=4) = %x", parent.GetRootHashRaw()) + if err := parent.AddLeaf(big.NewInt(0b1_0000), c0.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + if err := parent.AddLeaf(big.NewInt(0b1_1111), cF.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + parentRoot := parent.GetRootHashRaw() + canon := canonicalRoot(t, map[string][]byte{string(keyA): valA, string(keyB): valB}) + t.Logf("canonical RSMT root = %x", canon) + t.Logf("parent (certified) = %x", parentRoot) + if bytes.Equal(canon, parentRoot) { + t.Errorf("unexpectedly equal") + } + + frag, err := parent.GetShardInclusionFragment(0b1_0000) + if err != nil { + t.Fatal(err) + } + childCert, err := c0.GetInclusionCert(keyA) + if err != nil { + t.Fatal(err) + } + composed, err := api.ComposeInclusionCert(frag, childCert, c0.GetRootHashRaw()) + if err != nil { + t.Fatal(err) + } + depths := []int{} + for d := 0; d < 256; d++ { + if api.KeyBitBE(composed.Bitmap[:], d) == 1 { + depths = append(depths, d) + } + } + t.Logf("composed cert junction depths = %v (canonical would be [0])", depths) + zeros := 0 + for i, s := range composed.Siblings { + t.Logf(" sibling[%d] = %s", i, hex.EncodeToString(s[:])) + if bytes.Equal(s[:], make([]byte, 32)) { + zeros++ + } + } + t.Logf("all-zero siblings shipped to client: %d", zeros) + if err := composed.Verify(keyA, valA, parentRoot, api.SHA256); err != nil { + t.Errorf("composed cert must verify against certified root: %v", err) + } +} + +// Case 4: empty-tree root, standalone vs parent mode. +func TestLens_EmptyTreeRoots(t *testing.T) { + t.Logf("empty standalone (256-bit) root = %x", NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits).GetRootHashRaw()) + for _, n := range []int{1, 2, 4, 8} { + t.Logf("empty parent root SHARD_ID_LENGTH=%2d = %x", n, NewParentSparseMerkleTree(api.SHA256, n).GetRootHashRaw()) + } +} diff --git a/internal/smt/zz_specaudit2_test.go b/internal/smt/zz_specaudit2_test.go new file mode 100644 index 00000000..eb123d71 --- /dev/null +++ b/internal/smt/zz_specaudit2_test.go @@ -0,0 +1,165 @@ +package smt + +import ( + "crypto/rand" + "math/big" + mrand "math/rand" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// Differential fuzz: random (possibly nonsense) certs, keys, values, roots. +// The impl verifier and the literal spec verifier must always agree. +func TestSpec_DifferentialFuzz(t *testing.T) { + rng := mrand.New(mrand.NewSource(0xC0FFEE)) + for iter := 0; iter < 20000; iter++ { + var bm [32]byte + nbits := rng.Intn(6) + for i := 0; i < nbits; i++ { + api.SetBitBE(bm[:], rng.Intn(256)) + } + pc := 0 + for d := 0; d < 256; d++ { + pc += int(api.KeyBitBE(bm[:], d)) + } + // sometimes deliberately mismatch the sibling count + n := pc + switch rng.Intn(4) { + case 0: + n = pc + 1 + case 1: + if pc > 0 { + n = pc - 1 + } + } + sibs := make([][32]byte, n) + for i := range sibs { + rand.Read(sibs[i][:]) + } + k := make([]byte, 32) + v := make([]byte, rng.Intn(40)) + root := make([]byte, 32) + rand.Read(k) + rand.Read(v) + rand.Read(root) + + cert := &api.InclusionCert{Bitmap: bm, Siblings: sibs} + implOK := cert.Verify(k, v, root, api.SHA256) == nil + + specSibs := make([][]byte, len(sibs)) + for i := range sibs { + specSibs[i] = sibs[i][:] + } + specOK := specVerifyInclusion(bm[:], specSibs, root, k, v) + if implOK != specOK { + t.Fatalf("iter %d DIVERGENCE: impl=%v spec=%v bitmap=%x nsibs=%d", iter, implOK, specOK, bm, n) + } + } +} + +// Forgery attempt: take a genuine cert for leaf A, and try to make it verify +// for a different key that shares the path structure. Spec forbids it because +// regions are derived from the queried key. +func TestSpec_RegionBindsQueriedKey(t *testing.T) { + tree, leaves := buildRandomTree(t, 128) + root := tree.GetRootHashRaw() + for _, l := range leaves[:8] { + cert, err := tree.GetInclusionCert(l.k) + if err != nil { + t.Fatal(err) + } + // flip a key bit at a depth BELOW the deepest junction: descent side + // unchanged, region changes only at depths > flipped bit. + deepest := -1 + for d := 0; d < 256; d++ { + if api.KeyBitBE(cert.Bitmap[:], d) == 1 { + deepest = d + } + } + if deepest >= 255 { + continue + } + bad := append([]byte(nil), l.k...) + bad[(deepest+1)/8] ^= 0x80 >> uint((deepest+1)%8) + if err := cert.Verify(bad, l.v, root, api.SHA256); err == nil { + t.Fatalf("impl ACCEPTED cert for a key differing below deepest junction (deepest=%d)", deepest) + } + } +} + +// Sharded composition: does the certificate handed to a client verify under +// the LITERAL spec verifier with the state id as key? +func TestSpec_ComposedShardCertUnderSpec(t *testing.T) { + const shardBits = 4 + parent := NewParentSparseMerkleTree(api.SHA256, shardBits) + + // shard 0b1010 -> shardID sentinel path 0b1_1010? Shard id encoding: + // sentinel-prefixed int, BitLen()-1 == shardBits. + shardID := api.ShardID(0b1_0110) // shard bits (path bit order) = 0,1,1,0 + child := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, shardID) + + // build keys that route to this shard + var kvs []kv + for len(kvs) < 5 { + k := make([]byte, 32) + rand.Read(k) + p, err := api.FixedBytesToPath(k, api.StateTreeKeyLengthBits) + if err != nil { + t.Fatal(err) + } + v := make([]byte, 32) + rand.Read(v) + if err := child.AddLeaf(p, v); err != nil { + continue // wrong shard + } + kvs = append(kvs, kv{k, v}) + } + childRoot := child.GetRootHashRaw() + + // publish child root into the parent tree + shardPath := big.NewInt(int64(shardID)) + if err := parent.AddLeaf(shardPath, childRoot); err != nil { + t.Fatal(err) + } + // fill the other shards with random roots so siblings are non-trivial + for s := 0; s < 1<= 0; d-- { + if specBit(bitmap, d) == 0 { + continue + } + j-- + s := siblings[j] + p := specRegion(k, d) + var hL, hR []byte + if specBit(k, d) == 1 { + hL, hR = s, h + } else { + hL, hR = h, s + } + h = specNodeHash(hL, hR, d, p) + } + return h +} + +// Spec-constructed certificates (arbitrary junction depth sets, including 0 +// and 255, arbitrary value lengths) must be accepted by the implementation +// verifier and survive a wire round-trip. +func TestSpec_ImplAcceptsSpecConstructedCerts(t *testing.T) { + rng := mrand.New(mrand.NewSource(42)) + for iter := 0; iter < 5000; iter++ { + var bm [32]byte + depths := map[int]bool{} + nd := rng.Intn(8) + if iter < 3 { + // force the extreme depths at least once each + depths[0] = true + depths[255] = true + } + for i := 0; i < nd; i++ { + depths[rng.Intn(256)] = true + } + ordered := make([]int, 0, len(depths)) + for d := 0; d < 256; d++ { + if depths[d] { + api.SetBitBE(bm[:], d) + ordered = append(ordered, d) + } + } + sibs := make([][32]byte, len(ordered)) + for i := range sibs { + rand.Read(sibs[i][:]) + } + k := make([]byte, 32) + rand.Read(k) + v := make([]byte, rng.Intn(64)) + rand.Read(v) + + specSibs := make([][]byte, len(sibs)) + for i := range sibs { + specSibs[i] = sibs[i][:] + } + root := specRoot(bm[:], specSibs, k, v) + + cert := &api.InclusionCert{Bitmap: bm, Siblings: sibs} + if err := cert.Verify(k, v, root, api.SHA256); err != nil { + t.Fatalf("iter %d: impl REJECTED a spec-valid cert (depths=%v): %v", iter, ordered, err) + } + + // wire round trip: bitmap[32] || s_1..s_n, total 32+32n bytes + wire, err := cert.MarshalBinary() + if err != nil { + t.Fatal(err) + } + if len(wire) != 32+32*len(sibs) { + t.Fatalf("iter %d: wire length %d, want %d", iter, len(wire), 32+32*len(sibs)) + } + if !bytes.Equal(wire[:32], bm[:]) { + t.Fatalf("iter %d: bitmap is not the first 32 wire bytes", iter) + } + var back api.InclusionCert + if err := back.UnmarshalBinary(wire); err != nil { + t.Fatalf("iter %d: round trip decode: %v", iter, err) + } + if err := back.Verify(k, v, root, api.SHA256); err != nil { + t.Fatalf("iter %d: decoded cert rejected: %v", iter, err) + } + + // trailing garbage / truncation must be rejected + if err := (&api.InclusionCert{}).UnmarshalBinary(append(append([]byte(nil), wire...), 0x00)); err == nil { + t.Fatalf("iter %d: decoder accepted 1 trailing byte", iter) + } + if err := (&api.InclusionCert{}).UnmarshalBinary(append(append([]byte(nil), wire...), make([]byte, 32)...)); err == nil { + t.Fatalf("iter %d: decoder accepted an extra 32-byte sibling", iter) + } + if err := (&api.InclusionCert{}).UnmarshalBinary(wire[:len(wire)-1]); err == nil { + t.Fatalf("iter %d: decoder accepted a truncated cert", iter) + } + } +} + +// A junction at depth 255 (keys differing only in the final bit) must be +// generated and verified correctly end to end. +func TestSpec_Depth255Junction(t *testing.T) { + tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) + kA := bytes.Repeat([]byte{0xAB}, 32) + kB := append([]byte(nil), kA...) + kB[31] ^= 0x01 // differ only at bit 255 + vA, vB := []byte("A"), []byte("B") + pA, _ := api.FixedBytesToPath(kA, 256) + pB, _ := api.FixedBytesToPath(kB, 256) + if err := tree.AddLeaf(pA, vA); err != nil { + t.Fatal(err) + } + if err := tree.AddLeaf(pB, vB); err != nil { + t.Fatal(err) + } + root := tree.GetRootHashRaw() + cert, err := tree.GetInclusionCert(kA) + if err != nil { + t.Fatal(err) + } + if api.KeyBitBE(cert.Bitmap[:], 255) != 1 || len(cert.Siblings) != 1 { + t.Fatalf("expected a single junction at depth 255, bitmap=%x n=%d", cert.Bitmap, len(cert.Siblings)) + } + bm, sibs := certToSpec(cert) + if !specVerifyInclusion(bm, sibs, root, kA, vA) { + t.Fatal("spec verifier rejected depth-255 cert") + } + // kA has bit 255 = 1 (0xAB) so kA is the right child, kB (0xAA) the left. + want := specNodeHash(specLeafHash(kB, vB), specLeafHash(kA, vA), 255, specRegion(kA, 255)) + if !bytes.Equal(root, want) { + t.Fatalf("root mismatch at depth 255:\n impl %x\n spec %x", root, want) + } +} diff --git a/internal/smt/zz_specaudit4_test.go b/internal/smt/zz_specaudit4_test.go new file mode 100644 index 00000000..0d311119 --- /dev/null +++ b/internal/smt/zz_specaudit4_test.go @@ -0,0 +1,51 @@ +package smt + +import ( + "crypto/rand" + "math/big" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// Probe: with only two of 16 shards populated, the canonical RSMT has exactly +// one junction (depth 0) on the path to either shard. Does the parent tree +// emit a compressed path, or does it commit phantom all-zero subtrees? +func TestProbe_ParentTreePhantomSubtrees(t *testing.T) { + const shardBits = 4 + parent := NewParentSparseMerkleTree(api.SHA256, shardBits) + + rA := make([]byte, 32) + rB := make([]byte, 32) + rand.Read(rA) + rand.Read(rB) + + // shard path bits 0..3 = 0,0,0,0 -> sentinel int 0b1_0000 = 16 + // shard path bits 0..3 = 1,1,1,1 -> sentinel int 0b1_1111 = 31 + if err := parent.AddLeaf(big.NewInt(16), rA); err != nil { + t.Fatal(err) + } + if err := parent.AddLeaf(big.NewInt(31), rB); err != nil { + t.Fatal(err) + } + + frag, err := parent.GetShardInclusionFragment(16) + if err != nil { + t.Fatal(err) + } + var cert api.InclusionCert + if err := cert.UnmarshalBinary(frag.CertificateBytes); err != nil { + t.Fatal(err) + } + depths := []int{} + for d := 0; d < 256; d++ { + if api.KeyBitBE(cert.Bitmap[:], d) == 1 { + depths = append(depths, d) + } + } + t.Logf("junction depths on path to shard 0000 with only 2/16 shards live: %v (siblings=%d)", depths, len(cert.Siblings)) + for i, s := range cert.Siblings { + t.Logf(" sibling[%d] = %x", i, s) + } + t.Logf("parent root = %x", parent.GetRootHashRaw()) +} diff --git a/internal/smt/zz_specaudit_test.go b/internal/smt/zz_specaudit_test.go new file mode 100644 index 00000000..89c5c8c3 --- /dev/null +++ b/internal/smt/zz_specaudit_test.go @@ -0,0 +1,254 @@ +package smt + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// --------------------------------------------------------------------------- +// Literal transcription of appendix-hashtrees.tex Sec. C.3.7.1 +// (rsmt_verify_inclusion) + C.3.2.1/C.3.2.2 (leaf/node hash) + C.3.6 (cert +// path bit numbering). Written from the paper only, no reuse of pkg/api. +// --------------------------------------------------------------------------- + +// bit d of a big-endian bit string: k[0] is MSB of byte 0. +func specBit(buf []byte, d int) byte { return (buf[d/8] >> (7 - uint(d)%8)) & 1 } + +func specPopcount(bm []byte) int { + n := 0 + for d := 0; d < 8*len(bm); d++ { + n += int(specBit(bm, d)) + } + return n +} + +//

for p = k[0..d): 256-bit BE string, p in the first d bits, rest zero. +func specRegion(k []byte, d int) []byte { + p := make([]byte, 32) + for i := 0; i < d; i++ { + if specBit(k, i) == 1 { + p[i/8] |= 0x80 >> (uint(i) % 8) + } + } + return p +} + +func specLeafHash(k, v []byte) []byte { + h := sha256.New() + h.Write([]byte{0x00}) + h.Write(k) + h.Write(v) + return h.Sum(nil) +} + +func specNodeHash(hL, hR []byte, d int, p []byte) []byte { + h := sha256.New() + h.Write([]byte{0x01}) + h.Write([]byte{byte(d)}) + h.Write(p) + h.Write(hL) + h.Write(hR) + return h.Sum(nil) +} + +func specVerifyInclusion(bitmap []byte, siblings [][]byte, rho, k, v []byte) bool { + if len(bitmap) != 32 || len(k) != 32 || len(rho) != 32 { + return false + } + if len(siblings) != specPopcount(bitmap) { + return false + } + h := specLeafHash(k, v) + j := len(siblings) + for d := 255; d >= 0; d-- { + if specBit(bitmap, d) == 0 { + continue + } + j-- + s := siblings[j] + if len(s) != 32 { + return false + } + p := specRegion(k, d) + var hL, hR []byte + if specBit(k, d) == 1 { + hL, hR = s, h + } else { + hL, hR = h, s + } + h = specNodeHash(hL, hR, d, p) + } + return j == 0 && bytes.Equal(h, rho) +} + +func certToSpec(c *api.InclusionCert) ([]byte, [][]byte) { + sibs := make([][]byte, len(c.Siblings)) + for i := range c.Siblings { + sibs[i] = append([]byte(nil), c.Siblings[i][:]...) + } + return append([]byte(nil), c.Bitmap[:]...), sibs +} + +// --------------------------------------------------------------------------- + +type kv struct{ k, v []byte } + +func buildRandomTree(t *testing.T, n int) (*SparseMerkleTree, []kv) { + t.Helper() + tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) + out := make([]kv, 0, n) + seen := map[string]bool{} + for len(out) < n { + k := make([]byte, 32) + if _, err := rand.Read(k); err != nil { + t.Fatal(err) + } + if seen[string(k)] { + continue + } + seen[string(k)] = true + v := make([]byte, 32) + if _, err := rand.Read(v); err != nil { + t.Fatal(err) + } + p, err := api.FixedBytesToPath(k, api.StateTreeKeyLengthBits) + if err != nil { + t.Fatal(err) + } + if err := tree.AddLeaf(p, v); err != nil { + t.Fatal(err) + } + out = append(out, kv{k, v}) + } + return tree, out +} + +// TestSpec_GeneratedCertsVerifyUnderSpec: every cert the implementation +// generates must be accepted by the literal spec verifier. +func TestSpec_GeneratedCertsVerifyUnderSpec(t *testing.T) { + for _, n := range []int{1, 2, 3, 7, 33, 200} { + tree, leaves := buildRandomTree(t, n) + root := tree.GetRootHashRaw() + for _, l := range leaves { + cert, err := tree.GetInclusionCert(l.k) + if err != nil { + t.Fatalf("n=%d GetInclusionCert: %v", n, err) + } + bm, sibs := certToSpec(cert) + if !specVerifyInclusion(bm, sibs, root, l.k, l.v) { + t.Fatalf("n=%d: spec verifier REJECTED an implementation-generated cert for key %x", n, l.k) + } + if err := cert.Verify(l.k, l.v, root, api.SHA256); err != nil { + t.Fatalf("n=%d: impl verifier rejected own cert: %v", n, err) + } + } + } +} + +// TestSpec_ImplAgreesWithSpecOnMutations: for a corpus of mutated certs the +// impl verifier and the spec verifier must give the same answer. +func TestSpec_ImplAgreesWithSpecOnMutations(t *testing.T) { + tree, leaves := buildRandomTree(t, 64) + root := tree.GetRootHashRaw() + + type tc struct { + name string + mut func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) + } + cases := []tc{ + {"unchanged", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { return c, k, v }}, + {"extra sibling appended (no bitmap bit)", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + d := *c + d.Siblings = append(append([][32]byte{}, c.Siblings...), [32]byte{}) + return &d, k, v + }}, + {"sibling dropped (bitmap unchanged)", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + if len(c.Siblings) == 0 { + return c, k, v + } + d := *c + d.Siblings = append([][32]byte{}, c.Siblings[:len(c.Siblings)-1]...) + return &d, k, v + }}, + {"bitmap bit flipped on at unused depth", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + d := *c + for depth := 255; depth >= 0; depth-- { + if api.KeyBitBE(d.Bitmap[:], depth) == 0 { + api.SetBitBE(d.Bitmap[:], depth) + break + } + } + return &d, k, v + }}, + {"siblings reversed", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + d := *c + s := append([][32]byte{}, c.Siblings...) + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } + d.Siblings = s + return &d, k, v + }}, + {"wrong value", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + w := append([]byte(nil), v...) + w[0] ^= 0xff + return c, k, w + }}, + {"wrong key", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { + w := append([]byte(nil), k...) + w[31] ^= 0x01 + return c, w, v + }}, + } + + for _, l := range leaves[:16] { + base, err := tree.GetInclusionCert(l.k) + if err != nil { + t.Fatal(err) + } + for _, c := range cases { + mc, k, v := c.mut(base, l.k, l.v) + bm, sibs := certToSpec(mc) + specOK := specVerifyInclusion(bm, sibs, root, k, v) + implOK := mc.Verify(k, v, root, api.SHA256) == nil + if specOK != implOK { + t.Errorf("DIVERGENCE [%s] key=%x: spec=%v impl=%v", c.name, l.k, specOK, implOK) + } + } + } +} + +// TestSpec_RootHashMatchesSpecReconstruction: recompute the tree root purely +// from the spec (leaf hashes + certificate paths) for a 2-leaf tree. +func TestSpec_TwoLeafRootFromSpec(t *testing.T) { + tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) + kA := make([]byte, 32) + kB := make([]byte, 32) + kA[0] = 0x00 // bit0=0 + kB[0] = 0x80 // bit0=1 + vA := []byte("a") + vB := []byte("b") + pA, _ := api.FixedBytesToPath(kA, 256) + pB, _ := api.FixedBytesToPath(kB, 256) + if err := tree.AddLeaf(pA, vA); err != nil { + t.Fatal(err) + } + if err := tree.AddLeaf(pB, vB); err != nil { + t.Fatal(err) + } + root := tree.GetRootHashRaw() + // spec: junction at depth 0, region = empty (all-zero 32 bytes) + want := specNodeHash(specLeafHash(kA, vA), specLeafHash(kB, vB), 0, make([]byte, 32)) + if !bytes.Equal(root, want) { + t.Fatalf("root mismatch\n impl %x\n spec %x", root, want) + } + certA, err := tree.GetInclusionCert(kA) + if err != nil { + t.Fatal(err) + } + t.Logf("certA bitmap=%x siblings=%d", certA.Bitmap, len(certA.Siblings)) +} diff --git a/pkg/api/zz_audit_networkid_test.go b/pkg/api/zz_audit_networkid_test.go new file mode 100644 index 00000000..5c568fb6 --- /dev/null +++ b/pkg/api/zz_audit_networkid_test.go @@ -0,0 +1,31 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/unicitynetwork/bft-go-base/types" +) + +// AUDIT: platform.tex:456 requires ensure(UC.C^r.alpha = T.alpha). The seal here +// claims NetworkTestNet while the trust base is NetworkMainNet; the same root +// key signs both, so only an explicit network-ID comparison can catch it. +func TestAudit_NetworkIDNotChecked(t *testing.T) { + orig := auditSealNetworkID + auditSealNetworkID = types.NetworkTestNet + defer func() { auditSealNetworkID = orig }() + + proof, req, partitionID, tb, sid0, _ := buildProofCommittedInShard( + t, "1111111111111111111111111111111111111111111111111111111111111111", 0) + require.Equal(t, types.NetworkMainNet, tb.GetNetworkID()) + + var uc types.UnicityCertificate + require.NoError(t, types.Cbor.Unmarshal(proof.UnicityCertificate, &uc)) + t.Logf("seal.NetworkID=%d trustbase.NetworkID=%d", uc.UnicitySeal.NetworkID, tb.GetNetworkID()) + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: partitionID, ExpectedShardID: sid0, + }) + t.Logf("Verify() returned: %v", err) + require.NoError(t, err, "AUDIT: cross-network UC accepted") +} diff --git a/pkg/api/zz_audit_shardbinding_test.go b/pkg/api/zz_audit_shardbinding_test.go new file mode 100644 index 00000000..bb2bf8de --- /dev/null +++ b/pkg/api/zz_audit_shardbinding_test.go @@ -0,0 +1,157 @@ +package api + +import ( + "crypto" + "testing" + + "github.com/stretchr/testify/require" + test "github.com/unicitynetwork/bft-go-base/testutils" + testsig "github.com/unicitynetwork/bft-go-base/testutils/sig" + "github.com/unicitynetwork/bft-go-base/types" +) + +// auditSealNetworkID is the network ID stamped into the UnicitySeal; the trust +// base is always built for types.NetworkMainNet. +var auditSealNetworkID = types.NetworkMainNet + +// buildProofCommittedInShard is buildSignedSingleLeafProof, except the caller +// chooses which shard's InputRecord actually carries the single-leaf SMT root. +// That lets us commit a key whose canonical shard is sid0 inside sid1's tree. +func buildProofCommittedInShard(t *testing.T, stateIDHex string, committedIn int) ( + *InclusionProofV2, *CertificationRequest, types.PartitionID, types.RootTrustBase, + types.ShardID, types.ShardID, +) { + t.Helper() + + stateID := RequireNewImprintV2(stateIDHex) + txHash := RequireNewImprintV2("2222222222222222222222222222222222222222222222222222222222222222") + + req := &CertificationRequest{ + StateID: stateID, + CertificationData: CertificationData{TransactionHash: txHash}, + } + + key, err := stateID.GetTreeKey() + require.NoError(t, err) + const referenceTime uint64 = 1755000000 + hasher := NewDataHasher(InclusionProofV2HashAlgorithm) + hasher.Reset(). + AddData([]byte{0x00}). + AddData(key). + AddData(LeafValue(txHash.DataBytes(), referenceTime)) + leafRoot := append([]byte(nil), hasher.GetHash().RawHash...) + + cert := &InclusionCert{} + certBytes, err := cert.MarshalBinary() + require.NoError(t, err) + + sid0, sid1 := types.ShardID{}.Split() + const partitionID types.PartitionID = 0x0f0f0f0f + + mkIR := func(h []byte, tag byte) *types.InputRecord { + return &types.InputRecord{ + Version: 1, PreviousHash: []byte{0, 0, tag}, Hash: h, + BlockHash: []byte{0, 0, tag + 1}, SummaryValue: []byte{0, 0, tag + 2}, + Timestamp: types.NewTimestamp(), RoundNumber: 1, + } + } + var ir0, ir1 *types.InputRecord + if committedIn == 0 { + ir0, ir1 = mkIR(leafRoot, 1), mkIR(test.RandomBytes(32), 5) + } else { + ir0, ir1 = mkIR(test.RandomBytes(32), 1), mkIR(leafRoot, 5) + } + trHash0 := test.RandomBytes(32) + trHash1 := test.RandomBytes(32) + + sTree, err := types.CreateShardTree( + types.ShardingScheme{sid0, sid1}, + []types.ShardTreeInput{ + {Shard: sid0, IR: ir0, TRHash: trHash0}, + {Shard: sid1, IR: ir1, TRHash: trHash1}, + }, crypto.SHA256) + require.NoError(t, err) + + ownerShard, ownerIR, ownerTR := sid0, ir0, trHash0 + if committedIn == 1 { + ownerShard, ownerIR, ownerTR = sid1, ir1, trHash1 + } + stCert, err := sTree.Certificate(ownerShard) + require.NoError(t, err) + + ut, err := types.NewUnicityTree(crypto.SHA256, []*types.UnicityTreeData{{ + Partition: partitionID, ShardTreeRoot: sTree.RootHash(), + }}) + require.NoError(t, err) + utCert, err := ut.Certificate(partitionID) + require.NoError(t, err) + + signer, verifier := testsig.CreateSignerAndVerifier(t) + sigKey, err := verifier.MarshalPublicKey() + require.NoError(t, err) + tb, err := types.NewTrustBase(types.NetworkMainNet, []*types.NodeInfo{ + {NodeID: "test", SigKey: sigKey, Stake: 1}, + }) + require.NoError(t, err) + + seal := &types.UnicitySeal{ + Version: 1, NetworkID: auditSealNetworkID, RootChainRoundNumber: 1, + Timestamp: types.NewTimestamp(), + PreviousHash: test.RandomBytes(32), Hash: ut.RootHash(), + } + require.NoError(t, seal.Sign("test", signer)) + + ucBytes, err := types.Cbor.Marshal(types.UnicityCertificate{ + Version: 1, InputRecord: ownerIR, TRHash: ownerTR, + ShardTreeCertificate: stCert, UnicityTreeCertificate: utCert, UnicitySeal: seal, + }) + require.NoError(t, err) + + certifiedAt := referenceTime + proof := &InclusionProofV2{ + CertificationData: &req.CertificationData, + ReferenceTime: &certifiedAt, + CertificateBytes: certBytes, + UnicityCertificate: ucBytes, + } + return proof, req, partitionID, tb, sid0, sid1 +} + +// AUDIT: state ID 0x11.. has MSB 0, so f_SH(sid) = shard "0". The leaf is +// nevertheless committed in shard "1"'s SMT and certified by shard "1"'s UC. +// Per platform.tex:459 VerifyInclusionProof MUST reject. Go accepts. +func TestAudit_ForeignShardKeyAccepted(t *testing.T) { + proof, req, partitionID, tb, sid0, sid1 := buildProofCommittedInShard( + t, "1111111111111111111111111111111111111111111111111111111111111111", 1) + + key, err := req.StateID.GetTreeKey() + require.NoError(t, err) + t.Logf("key[0]=%08b f_SH(sid)=%q (sid0=%q, sid1=%q)", key[0], sid0.String(), sid0.String(), sid1.String()) + require.True(t, sid0.Comparator()(key), "key must route to shard 0") + require.False(t, sid1.Comparator()(key), "key must NOT route to shard 1") + + err = proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid1, // the shard that served the proof + }) + t.Logf("Verify() returned: %v", err) + require.NoError(t, err, "AUDIT: Go verifier accepted a foreign-shard key") +} + +// AUDIT: the mandatory H(CD_beta) = UC.C^uni.dhash binding is skippable. +func TestAudit_ShardConfHashOptional(t *testing.T) { + proof, req, partitionID, tb, sid0, _ := buildProofCommittedInShard( + t, "1111111111111111111111111111111111111111111111111111111111111111", 0) + + // UC.ShardConfHash was never set (nil) yet verification passes when the + // verifier context leaves ShardConfHash nil. + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + ShardConfHash: nil, + }) + t.Logf("Verify() with nil ShardConfHash returned: %v", err) + require.NoError(t, err) +} diff --git a/pkg/api/zz_xcheck_shardbind_test.go b/pkg/api/zz_xcheck_shardbind_test.go new file mode 100644 index 00000000..acfd3c7f --- /dev/null +++ b/pkg/api/zz_xcheck_shardbind_test.go @@ -0,0 +1,208 @@ +package api + +import ( + "crypto" + "testing" + + "github.com/stretchr/testify/require" + test "github.com/unicitynetwork/bft-go-base/testutils" + testsig "github.com/unicitynetwork/bft-go-base/testutils/sig" + "github.com/unicitynetwork/bft-go-base/types" +) + +// xcheckBuild builds a fully signed v2 inclusion proof for a two-shard +// partition (SH = {"0","1"}) in which the leaf for `stateIDHex` is committed +// into `committingShard`'s SMT -- regardless of which shard f_SH(sid) actually +// names. netID is written into the seal; tbNet into the trust base. +// +// Independent of buildSignedSingleLeafProof in inclusion_proof_v2_verify_test.go: +// that helper always puts the leaf root in shard 0's IR, so it cannot express a +// foreign-shard commitment. +func xcheckBuild(t *testing.T, stateIDHex string, committingShard types.ShardID, + netID types.NetworkID, tbNet types.NetworkID) ( + *InclusionProofV2, *CertificationRequest, types.PartitionID, types.RootTrustBase) { + t.Helper() + + stateID := RequireNewImprintV2(stateIDHex) + txHash := RequireNewImprintV2("2222222222222222222222222222222222222222222222222222222222222222") + req := &CertificationRequest{ + StateID: stateID, + CertificationData: CertificationData{TransactionHash: txHash}, + } + + key, err := stateID.GetTreeKey() + require.NoError(t, err) + const referenceTime uint64 = 1755000000 + + // Single-leaf RSMT root: H(0x00 || key || v). + h := NewDataHasher(InclusionProofV2HashAlgorithm) + h.Reset(). + AddData([]byte{0x00}). + AddData(key). + AddData(LeafValue(txHash.DataBytes(), referenceTime)) + leafRoot := append([]byte(nil), h.GetHash().RawHash...) + + cert := &InclusionCert{} // single leaf, no siblings + certBytes, err := cert.MarshalBinary() + require.NoError(t, err) + + sid0, sid1 := types.ShardID{}.Split() + const partitionID types.PartitionID = 0x0f0f0f0f + + mkIR := func(hash []byte, salt byte) *types.InputRecord { + return &types.InputRecord{ + Version: 1, PreviousHash: []byte{0, 0, salt}, Hash: hash, + BlockHash: []byte{0, 0, salt + 1}, SummaryValue: []byte{0, 0, salt + 2}, + Timestamp: types.NewTimestamp(), RoundNumber: 1, Epoch: 0, + } + } + // The committing shard's IR carries the leaf root; the other shard is filler. + ir0, ir1 := mkIR(test.RandomBytes(32), 1), mkIR(test.RandomBytes(32), 5) + if committingShard.Equal(sid0) { + ir0 = mkIR(leafRoot, 1) + } else { + ir1 = mkIR(leafRoot, 5) + } + trHash0, trHash1 := test.RandomBytes(32), test.RandomBytes(32) + + sTree, err := types.CreateShardTree( + types.ShardingScheme{sid0, sid1}, + []types.ShardTreeInput{ + {Shard: sid0, IR: ir0, TRHash: trHash0}, + {Shard: sid1, IR: ir1, TRHash: trHash1}, + }, crypto.SHA256) + require.NoError(t, err) + + ownerIR, ownerTR := ir0, trHash0 + if committingShard.Equal(sid1) { + ownerIR, ownerTR = ir1, trHash1 + } + stCert, err := sTree.Certificate(committingShard) + require.NoError(t, err) + + ut, err := types.NewUnicityTree(crypto.SHA256, []*types.UnicityTreeData{ + {Partition: partitionID, ShardTreeRoot: sTree.RootHash()}, + }) + require.NoError(t, err) + utCert, err := ut.Certificate(partitionID) + require.NoError(t, err) + + signer, verifier := testsig.CreateSignerAndVerifier(t) + sigKey, err := verifier.MarshalPublicKey() + require.NoError(t, err) + tb, err := types.NewTrustBase(tbNet, []*types.NodeInfo{{NodeID: "n1", SigKey: sigKey, Stake: 1}}) + require.NoError(t, err) + + seal := &types.UnicitySeal{ + Version: 1, NetworkID: netID, RootChainRoundNumber: 1, + Timestamp: types.NewTimestamp(), PreviousHash: test.RandomBytes(32), + Hash: ut.RootHash(), + } + require.NoError(t, seal.Sign("n1", signer)) + + ucBytes, err := types.Cbor.Marshal(types.UnicityCertificate{ + Version: 1, InputRecord: ownerIR, TRHash: ownerTR, + ShardTreeCertificate: stCert, UnicityTreeCertificate: utCert, UnicitySeal: seal, + }) + require.NoError(t, err) + + rt := referenceTime + return &InclusionProofV2{ + CertificationData: &req.CertificationData, + ReferenceTime: &rt, + CertificateBytes: certBytes, + UnicityCertificate: ucBytes, + }, req, partitionID, tb +} + +// Establish the routing fact first: sid 0x1111... routes to shard "0". +func TestXCheck_RoutingFact(t *testing.T) { + sid0, sid1 := types.ShardID{}.Split() + key, err := RequireNewImprintV2( + "1111111111111111111111111111111111111111111111111111111111111111").GetTreeKey() + require.NoError(t, err) + require.True(t, sid0.Comparator()(key), "f_SH(sid) must be shard 0") + require.False(t, sid1.Comparator()(key), "sid must NOT route to shard 1") + t.Logf("key[0]=%#02x -> f_SH = shard %q (not %q)", key[0], sid0, sid1) +} + +// THE CLAIM: a leaf whose f_SH(sid) = "0" but which shard "1" committed and +// certified under a fully valid UC is accepted by InclusionProofV2.Verify. +func TestXCheck_ForeignShardLeafAccepted(t *testing.T) { + sid0, sid1 := types.ShardID{}.Split() + proof, req, pid, tb := xcheckBuild(t, + "1111111111111111111111111111111111111111111111111111111111111111", // f_SH = sid0 + sid1, // but committed + certified by shard 1 + types.NetworkMainNet, types.NetworkMainNet) + + // Caller derives ExpectedShardID from the serving endpoint / the proof's own + // UC -- the in-repo pattern -- i.e. sid1. + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: pid, ExpectedShardID: sid1, + }) + t.Logf("Verify(ExpectedShardID=sid1) for a sid0-routed key => %v", err) + require.NoError(t, err, "spec ensure(f_SH(sid)=sigma) would have rejected this") + + // Control: same construction with the key committed in its correct shard. + p2, r2, pid2, tb2 := xcheckBuild(t, + "1111111111111111111111111111111111111111111111111111111111111111", + sid0, types.NetworkMainNet, types.NetworkMainNet) + require.NoError(t, p2.Verify(r2, &VerifierContext{ + TrustBase: tb2, PartitionID: pid2, ExpectedShardID: sid0, + })) +} + +// Control the other way: the ONLY shard check is equality with the caller's +// value, so passing the true responsible shard rejects the honest proof too. +func TestXCheck_OnlyCheckIsCallerEquality(t *testing.T) { + sid0, sid1 := types.ShardID{}.Split() + proof, req, pid, tb := xcheckBuild(t, + "1111111111111111111111111111111111111111111111111111111111111111", + sid1, types.NetworkMainNet, types.NetworkMainNet) + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, + }) + require.ErrorContains(t, err, "invalid shard ID") + t.Logf("ExpectedShardID=sid0 => %v (equality with caller value, not f_SH)", err) +} + +// UC.C^r.alpha = T.alpha is unchecked: a testnet-sealed UC verifies against a +// mainnet trust base. +func TestXCheck_NetworkIDUnchecked(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + proof, req, pid, tb := xcheckBuild(t, + "1111111111111111111111111111111111111111111111111111111111111111", + sid0, types.NetworkTestNet, types.NetworkMainNet) + require.EqualValues(t, types.NetworkMainNet, tb.GetNetworkID()) + + var uc types.UnicityCertificate + require.NoError(t, types.Cbor.Unmarshal(proof.UnicityCertificate, &uc)) + require.EqualValues(t, types.NetworkTestNet, uc.UnicitySeal.NetworkID) + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, + }) + t.Logf("seal.NetworkID=%d vs trustBase.NetworkID=%d => Verify: %v", + uc.UnicitySeal.NetworkID, tb.GetNetworkID(), err) + require.NoError(t, err, "spec ensure(UC.C^r.alpha = T.alpha) would have rejected this") +} + +// ShardConfHash: the UC carries one, the verifier context does not, and the +// mismatch is simply not looked at. +func TestXCheck_ShardConfHashSkippable(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + proof, req, pid, tb := xcheckBuild(t, + "1111111111111111111111111111111111111111111111111111111111111111", + sid0, types.NetworkMainNet, types.NetworkMainNet) + require.NoError(t, proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, ShardConfHash: nil, + })) + // Supplying any non-nil value does get compared -- so the check exists but + // is opt-in, and no production caller opts in. + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, + ShardConfHash: []byte{0xAA}, + }) + require.ErrorContains(t, err, "invalid shard configuration hash") + t.Logf("nil ShardConfHash: accepted; non-nil: %v", err) +} From 17e2ec0ae1049c84b12de2e2764354c7be07b713 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 20:27:17 +0200 Subject: [PATCH 08/12] docs: record the yellowpaper divergences instead of documenting them as design An audit against the yellowpaper found that this document had turned two implementation gaps into apparent specification. The shard binding. platform.tex VerifyInclusionProof takes the partition description as an input and requires f_SH(sid) = sigma -- the expected shard is derived from the KEY. It also states that VerifyUnicityCert does not by itself prove a state identifier belongs to the shard named in the certificate, and that the binding is the proof verifier'\''s job. InclusionProofV2.Verify instead compares the UC'\''s shard against a caller-supplied ExpectedShardID. A leaf routing to shard A, committed in shard B'\''s SMT under shard B'\''s validly signed UC, verifies -- reproduced in testing. The previous text told integrators to derive ExpectedShardID from configuration, which is not the specified check and does not close the gap. Now recorded as the soundness divergence it is. The network id check UC.C^r.alpha = T.alpha is likewise absent. ExclusionCert. appendix-hashtrees.tex orders the certificate bitmap || siblings || k'\'' || v'\''; the Go type puts the fixed 32-byte fields first, which also makes a variable-length v'\'' unencodable and leaves the spec'\''s empty-tree certificate undecodable. Nothing generates or verifies these, and neither security-critical check the spec names exists. Documented as a divergence rather than as a frozen format. The preamble now says plainly that the yellowpaper is authoritative and that this file must not present an implementation gap as a specification. --- docs/inclusion-proof-wire.md | 91 ++++++++++++++++++++++----- internal/smt/zz_audit_liveset_test.go | 87 +++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 16 deletions(-) create mode 100644 internal/smt/zz_audit_liveset_test.go diff --git a/docs/inclusion-proof-wire.md b/docs/inclusion-proof-wire.md index 9acfd4ac..ddd56c04 100644 --- a/docs/inclusion-proof-wire.md +++ b/docs/inclusion-proof-wire.md @@ -1,14 +1,14 @@ # Inclusion proof wire specification (v2) -Frozen wire format for `get_inclusion_proof.v2`. Three source comments cite this +Wire format for `get_inclusion_proof.v2`. Three source comments cite this document as normative: `pkg/api/types.go` (`InclusionProofV2`), and `pkg/api/inclusion_cert.go` (`InclusionCert`, `ExclusionCert`). Corresponds to the Unicity yellowpaper's inclusion proof -$\pi^{\mathsf{inc}} = (\mathsf{sid}, v, C^{\mathsf{inc}}, UC)$. Where this -document and the yellowpaper disagree about intent, the yellowpaper wins; where -they disagree about bytes, this document describes what the Go implementation -actually emits. +$\pi^{\mathsf{inc}} = (\mathsf{sid}, v, C^{\mathsf{inc}}, UC)$. **The yellowpaper +is authoritative.** This document describes what the Go implementation actually +emits, and where the two differ it says so explicitly and names the paper as +correct -- it does not present an implementation gap as a specification. ## CBOR tags @@ -38,9 +38,9 @@ The `result` field of `get_inclusion_proof.v2` is a hex-encoded CBOR array: **Discriminator.** `certificationData != null` ⇒ inclusion, and `certificateBytes` is an `InclusionCert`. `certificationData == null` ⇒ -non-inclusion, and `certificateBytes` is an `ExclusionCert`. Non-inclusion -verification is not implemented in Go; the codec is frozen so clients can decode -today. +non-inclusion, and `certificateBytes` is an `ExclusionCert`. Non-inclusion is +neither generated nor verified in Go, and the `ExclusionCert` layout below +diverges from the yellowpaper -- do not build against it yet. ### `CertificationData` @@ -101,15 +101,43 @@ outside it: Decoding rejects: fewer than 32 bytes (truncated), a remainder not a multiple of 32 (misaligned), and a sibling count disagreeing with the bitmap popcount. -## `ExclusionCert` +## `ExclusionCert` — diverges from the yellowpaper, and is unimplemented + +The Go type encodes: ``` k_l[32] || h_l[32] || bitmap[32] || s_1[32] || ... || s_n[32] ``` -`(k_l, h_l)` is the witness leaf present in the tree at the position reached when -routing the query key. `bitmap` and siblings describe the path from the root to -that position, under the same root-to-leaf ordering as `InclusionCert`. +`appendix-hashtrees.tex` specifies the **opposite order**: + +``` +bitmap[32] || s_1[32] || ... || s_n[32] || k'[32] || v' +``` + +These are not interchangeable: one logical certificate encodes to two different +byte strings, and the Go decoder rejects the spec layout with a bitmap/popcount +mismatch. The spec puts `v'` last so the remainder after the fixed-size terminal +key is the value; with the fixed 32-byte field leading, a variable-length `v'` is +structurally unencodable here. The aggregation profile does permit +`len(v') = 32`, so only the ordering diverges — but `h_l` names the leaf **value** +`v'`, not a hash of it, which the field name obscures. + +The spec's empty-tree certificate `C^exc_empty` (the empty byte string) is also +undecodable: `UnmarshalBinary(nil)` returns a truncation error, so a genesis tree +has no encodable certificate. + +**Nothing generates or verifies these.** `internal/smt` exposes only +`GetInclusionCert`; `ExclusionCert.Verify` returns `ErrExclusionNotImpl`; and a +non-inclusion response carries `certificateBytes` as CBOR null (`f6`) rather than +the spec's empty byte string (`40`). Neither of the two security-critical checks +the spec names — `k' ≠ k`, and `k[d] = k'[d]` at every junction depth, with the +region taken from the authenticated terminal key `k'` — exists in this repo. + +This is fail-closed: no forged absence proof is accepted because none is +accepted. But absence and not-yet-certified are indistinguishable on the wire, +and this layout should not be treated as frozen until it is reconciled with +`appendix-hashtrees.tex`. ## Hash rules @@ -160,7 +188,38 @@ child. The nil-guard error strings are part of the public contract so reference verifiers in other languages can pin them. -`Verify` does **not** check that `sid` routes to the expected shard, though -`api.MatchesShardPrefix` exists and the admission path applies it. A caller that -derives `ExpectedShardID` from the proof's own UC would accept a leaf certified -by the wrong shard; derive it from configuration instead. +## Known divergence from the yellowpaper: the shard binding is not checked + +**This is a soundness gap, not a caller responsibility.** An earlier revision of +this document told integrators to "derive `ExpectedShardID` from configuration". +That is not the specified check and does not close the hole. + +`platform.tex` `VerifyInclusionProof` takes the partition description `CD_β` as +an input and mandates, before any tree check: + +``` +ensure(UC.C^r.α = T.α) +σ ← UC.C^shard.σ +ensure(σ ∈ CD_β.SH) +ensure(f_{CD_β.SH}(sid) = σ) // Proof comes from the right shard +``` + +`f_SH` derives the expected shard **from the key itself**. `platform.tex` +explicitly forecloses delegating this to certificate verification: +`VerifyUnicityCert` "does not, by itself, prove that a particular state +identifier belongs to the shard named in `C^shard`; that binding is checked by +the proof verification functions below." + +`InclusionProofV2.Verify` implements the tree and certificate steps but not the +binding: it compares the UC's shard against a caller-supplied +`VerifierContext.ExpectedShardID` rather than computing `f_SH(sid)`. In a +multi-shard deployment a leaf whose key routes to shard A, committed in shard +B's SMT under shard B's validly signed UC, therefore verifies — reproduced in +testing. That is cross-shard double-spend exposure. The network id +`UC.C^r.α = T.α` is likewise unchecked, so a certificate sealed for one network +verifies against another network's trust base. + +Until `VerifierContext` carries the sharding scheme and `Verify` derives the +expected shard from `sid`, do not rely on this function alone for cross-shard +safety. `api.MatchesShardPrefix` implements `f_SH` and the admission path +applies it correctly. diff --git a/internal/smt/zz_audit_liveset_test.go b/internal/smt/zz_audit_liveset_test.go new file mode 100644 index 00000000..9926cfcb --- /dev/null +++ b/internal/smt/zz_audit_liveset_test.go @@ -0,0 +1,87 @@ +package smt + +import ( + "fmt" + "math/big" + "testing" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// Same authenticated leaf set (one live commitment in shard 0b10000, nothing +// anywhere else), two different IR.h values depending only on whether an +// empty shard happened to report in this round. +func TestAudit_RootDependsOnLiveShardSet(t *testing.T) { + const shardIDLen = 4 + + emptyChild := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, api.ShardID(0b11111)) + fmt.Printf("empty CHILD shard root = %x\n", emptyChild.GetRootHashRaw()) + + realRoot := make([]byte, 32) + for i := range realRoot { + realRoot[i] = 0xAA + } + + p1 := NewParentSparseMerkleTree(api.SHA256, shardIDLen) + if err := p1.AddLeaf(big.NewInt(0b10000), realRoot); err != nil { + t.Fatal(err) + } + fmt.Printf("IR.h with shard 1111 silent = %x\n", p1.GetRootHashRaw()) + + p2 := NewParentSparseMerkleTree(api.SHA256, shardIDLen) + if err := p2.AddLeaf(big.NewInt(0b10000), realRoot); err != nil { + t.Fatal(err) + } + if err := p2.AddLeaf(big.NewInt(0b11111), emptyChild.GetRootHashRaw()); err != nil { + t.Fatal(err) + } + fmt.Printf("IR.h with shard 1111 reporting 0 = %x\n", p2.GetRootHashRaw()) + + // dependence on SHARD_ID_LENGTH for the *empty* tree, which is certified + // verbatim by processRound when no shard reported this round. + for _, kl := range []int{1, 2, 3, 4, 8, 16} { + tr := NewParentSparseMerkleTree(api.SHA256, kl) + fmt.Printf("empty parent IR.h @ SHARD_ID_LENGTH=%2d -> %x\n", kl, tr.GetRootHashRaw()) + } +} + +// Cleanest canonicity counterexample: exactly one live shard. The +// whole-partition RSMT root then equals that shard's own root (all keys share +// the shard prefix, so the child tree IS the partition tree). +func TestAudit_SingleLiveShardRootIsNotChildRoot(t *testing.T) { + realRoot := make([]byte, 32) + for i := range realRoot { + realRoot[i] = 0xAA + } + for _, kl := range []int{1, 2, 4} { + p := NewParentSparseMerkleTree(api.SHA256, kl) + sid := big.NewInt(int64(1< Date: Tue, 25 Aug 2026 20:30:12 +0200 Subject: [PATCH 09/12] docs: the request timeout is optional per the yellowpaper, not a migration case Yellowpaper ba5d716 ("us request expiry as absolute time") specifies the request timeout as optional: Q carries tau_Q_bar in T union {bottom}, bottom is written as CBOR null at a fixed position, and the effective timeout is tau_a + Delta when absent and tau_Q_bar otherwise, where tau_a is the latest consensus-derived reference time at admission and Delta the service's default request lifetime. The assigned value is service metadata and does not alter txhash. That is what this service already does, so the comments claiming the paper makes the deadline mandatory were wrong, as was framing the deadline-origin counter as a migration backlog with retirement of the absent form as its goal. The counter stays -- what share of traffic depends on DEFAULT_REQUEST_TTL is worth seeing -- but it is operational visibility, not a deprecation clock. Appendix ba5d716 also confirms the fixed-shape encoding this wire already uses: "Each version of a structure therefore has exactly one tuple shape and one element count, and an optional element occupies its position whether or not it carries a value." Also conformant, and now stated as such: the admission check is permitted but not sufficient, with the authoritative check at leaf materialisation against that round's pinned reference time -- which is where it already runs. --- internal/metrics/metrics.go | 14 +++++++----- internal/service/block_records_shape_test.go | 6 ++--- internal/service/service.go | 24 ++++++++++---------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index fba6675f..1721a5e1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -188,10 +188,12 @@ var ( CommitmentsDroppedRejected = CommitmentsDroppedTotal.WithLabelValues("rejected") // CertificationRequestsByDeadline splits accepted requests by whether the - // requester supplied an exclusive deadline or the service assigned one. - // The yellowpaper makes the deadline a mandatory element of the request; the - // service_assigned series is the migration backlog, and reaching zero is the - // precondition for rejecting requests that omit it. + // requester supplied an exclusive deadline or the service assigned one from + // DEFAULT_REQUEST_TTL. Both forms are specified -- the yellowpaper's request + // timeout is optional -- so this is operational visibility, not a migration + // counter: it shows what share of traffic depends on the service's default + // lifetime, which is the knob that decides how long those requests stay + // admissible. CertificationRequestsByDeadline = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aggregator_certification_requests_by_deadline_total", @@ -201,8 +203,8 @@ var ( ) // Resolved once, as for the drop reasons above. Increment these only after - // the request is actually accepted -- an expired or duplicate request is not - // part of the migration backlog. + // the request is actually accepted -- an expired or duplicate request was + // never admitted and must not be counted. DeadlineOriginExplicit = CertificationRequestsByDeadline.WithLabelValues("explicit") DeadlineOriginServiceAssigned = CertificationRequestsByDeadline.WithLabelValues("service_assigned") diff --git a/internal/service/block_records_shape_test.go b/internal/service/block_records_shape_test.go index 8da5690a..abf69c42 100644 --- a/internal/service/block_records_shape_test.go +++ b/internal/service/block_records_shape_test.go @@ -84,9 +84,9 @@ func keysOf(m map[string]any) []string { return out } -// The deadline-origin counter is the migration signal for retiring absent -// deadlines, so it must count only requests that were actually accepted. An -// expired request is rejected and must not inflate the backlog. +// The deadline-origin counter reports the share of traffic relying on the +// service-assigned deadline, so it must count only requests that were actually +// accepted. An expired request is rejected and must not be counted. func TestDeadlineOriginCountsOnlyAcceptedRequests(t *testing.T) { ctx := context.Background() log, err := logger.New("error", "text", "stdout", false) diff --git a/internal/service/service.go b/internal/service/service.go index 42dbd84c..69130e46 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -195,17 +195,17 @@ func (as *AggregatorService) CertificationRequest(ctx context.Context, req *api. if referenceTime == 0 { return &api.CertificationResponse{Status: api.CertificationStatusServiceNotReady}, nil } - // An explicit deadline is used verbatim and is covered by the witness. When - // the requester omitted one, the service derives a deadline from consensus - // reference time; that value is service metadata and is never recorded in the - // leaf, signed, or checked by a later verifier. + // This implements the yellowpaper's effective timeout: for a request + // Q = (predicate, sourceStateHash, txhash, tau_Q_bar, u) with an optional + // tau_Q_bar, the effective timeout is tau_a + Delta when the requester + // omitted one and tau_Q_bar otherwise, where tau_a is the latest + // consensus-derived reference time at admission and Delta the service's + // default request lifetime (DEFAULT_REQUEST_TTL). The assigned value is + // service metadata: it does not alter txhash and is not recorded in the leaf. // - // The yellowpaper defines the request as Q = (predicate, sourceStateHash, - // txhash, tau_Q, u) with tau_Q mandatory, and makes "tau < tau_Q" a step of - // verifying a certified transaction. A request certified without a deadline - // therefore leaves a later verifier unable to perform that check at all, - // rather than merely choosing not to. The absent form is accepted for - // migration; the counter below is what tells us when it can be retired. + // The check below is the admission check, which the paper permits but does + // not accept as sufficient -- the authoritative one runs at leaf + // materialisation against that round's pinned reference time. var effectiveTimeout uint64 deadlineOrigin := metrics.DeadlineOriginExplicit if expiresAt := req.CertificationData.ExpiresAt; expiresAt != nil { @@ -252,8 +252,8 @@ func (as *AggregatorService) CertificationRequest(ctx context.Context, req *api. } // Counted here rather than at assignment: expired, duplicate and - // failed-to-store requests are not accepted, and counting them would inflate - // the migration backlog with requests that never reach a leaf. + // failed-to-store requests are never admitted, and counting them would + // misreport the share of traffic relying on the service-assigned deadline. deadlineOrigin.Inc() as.logger.WithContext(ctx).Log(ctx, logger.LevelTrace, "CertificationData submitted successfully", "stateId", req.StateID) From 502a3d4ece12bc7b7cebe19ea495c0b2ca7a0ac3 Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 20:38:18 +0200 Subject: [PATCH 10/12] chore: remove audit scaffolding committed by mistake Eleven throwaway probe tests written by the spec-audit subagents in this worktree were swept into f031c4a and 17e2ec0 by a 'git add -A'. They are scratch reproductions, not tests meant to ship -- roughly 1,700 lines of zz_-prefixed files under internal/smt and pkg/api. Net diff against main is unaffected; they were added and are now removed. The most useful of them, the reproduction of the missing f_SH(sid) shard binding, is preserved in issue #183 rather than here, where it would have shipped as a permanently failing test. --- internal/smt/zz_audit_compose_test.go | 103 --------- internal/smt/zz_audit_liveset_test.go | 87 -------- internal/smt/zz_audit_parent_test.go | 150 ------------- internal/smt/zz_lens_parentcanon_test.go | 217 ------------------- internal/smt/zz_specaudit2_test.go | 165 --------------- internal/smt/zz_specaudit3_test.go | 144 ------------- internal/smt/zz_specaudit4_test.go | 51 ----- internal/smt/zz_specaudit_test.go | 254 ----------------------- pkg/api/zz_audit_networkid_test.go | 31 --- pkg/api/zz_audit_shardbinding_test.go | 157 -------------- pkg/api/zz_xcheck_shardbind_test.go | 208 ------------------- 11 files changed, 1567 deletions(-) delete mode 100644 internal/smt/zz_audit_compose_test.go delete mode 100644 internal/smt/zz_audit_liveset_test.go delete mode 100644 internal/smt/zz_audit_parent_test.go delete mode 100644 internal/smt/zz_lens_parentcanon_test.go delete mode 100644 internal/smt/zz_specaudit2_test.go delete mode 100644 internal/smt/zz_specaudit3_test.go delete mode 100644 internal/smt/zz_specaudit4_test.go delete mode 100644 internal/smt/zz_specaudit_test.go delete mode 100644 pkg/api/zz_audit_networkid_test.go delete mode 100644 pkg/api/zz_audit_shardbinding_test.go delete mode 100644 pkg/api/zz_xcheck_shardbind_test.go diff --git a/internal/smt/zz_audit_compose_test.go b/internal/smt/zz_audit_compose_test.go deleted file mode 100644 index 5372f71c..00000000 --- a/internal/smt/zz_audit_compose_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package smt - -import ( - "fmt" - "math/big" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// End-to-end: child shard cert + parent fragment -> composed cert verified by -// an end client against the parent UC.IR.h. Shows whether the all-zero -// phantom sibling is load-bearing in the client-verified path. -func TestAudit_ComposedCertCarriesZeroSibling(t *testing.T) { - const shardIDLen = 4 - - // ---- child aggregator for shard 0b10000 (shard prefix bits 0000) ---- - childShardID := api.ShardID(0b10000) - child := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, childShardID) - - key := make([]byte, 32) - // top 4 bits must be 0000 to live in this shard; set some lower bits - key[0] = 0x0A - key[1] = 0x5C - key[31] = 0x99 - path, err := api.FixedBytesToPath(key, api.StateTreeKeyLengthBits) - if err != nil { - t.Fatal(err) - } - value := []byte("leaf-value-bytes") - if err := child.AddLeaf(path, value); err != nil { - t.Fatal(err) - } - childRoot := child.GetRootHashRaw() - childCert, err := child.GetInclusionCert(key) - if err != nil { - t.Fatal(err) - } - fmt.Printf("childRoot = %x\n", childRoot) - - // ---- parent aggregator ---- - parent := NewParentSparseMerkleTree(api.SHA256, shardIDLen) - if err := parent.AddLeaf(big.NewInt(int64(childShardID)), childRoot); err != nil { - t.Fatal(err) - } - // a second live shard so the root isn't a degenerate unary chain - other := make([]byte, 32) - for i := range other { - other[i] = 0xBB - } - if err := parent.AddLeaf(big.NewInt(0b11111), other); err != nil { - t.Fatal(err) - } - parentRoot := parent.GetRootHashRaw() - fmt.Printf("parentRoot (= UC.IR.h) = %x\n", parentRoot) - - frag, err := parent.GetShardInclusionFragment(childShardID) - if err != nil { - t.Fatal(err) - } - - composed, err := api.ComposeInclusionCert(frag, childCert, childRoot) - if err != nil { - t.Fatal(err) - } - - zero := [32]byte{} - nZero := 0 - for i, s := range composed.Siblings { - if s == zero { - nZero++ - fmt.Printf("composed sibling[%d] is ALL-ZERO\n", i) - } - } - depths := []int{} - for d := 0; d < 256; d++ { - if api.KeyBitBE(composed.Bitmap[:], d) == 1 { - depths = append(depths, d) - } - } - fmt.Printf("composed bitmap depths = %v (siblings=%d, all-zero=%d)\n", - depths, len(composed.Siblings), nZero) - - if err := composed.Verify(key, value, parentRoot, api.SHA256); err != nil { - t.Fatalf("composed cert failed to verify: %v", err) - } - fmt.Println("composed cert VERIFIES against parent root -> zero sibling is load-bearing") - - // Sanity: dropping the all-zero sibling (as a canonical RSMT would) breaks it. - stripped := &api.InclusionCert{Bitmap: composed.Bitmap} - for i, s := range composed.Siblings { - if s == zero { - api.ClearSuffixBE(nil, 0) // no-op, keep import stable - // clear the corresponding bitmap bit - idx := depths[i] - stripped.Bitmap[idx/8] &^= 0x80 >> (uint(idx) % 8) - continue - } - stripped.Siblings = append(stripped.Siblings, s) - } - err = stripped.Verify(key, value, parentRoot, api.SHA256) - fmt.Printf("canonical (zero-sibling-compressed) cert verify -> %v\n", err) -} diff --git a/internal/smt/zz_audit_liveset_test.go b/internal/smt/zz_audit_liveset_test.go deleted file mode 100644 index 9926cfcb..00000000 --- a/internal/smt/zz_audit_liveset_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package smt - -import ( - "fmt" - "math/big" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// Same authenticated leaf set (one live commitment in shard 0b10000, nothing -// anywhere else), two different IR.h values depending only on whether an -// empty shard happened to report in this round. -func TestAudit_RootDependsOnLiveShardSet(t *testing.T) { - const shardIDLen = 4 - - emptyChild := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, api.ShardID(0b11111)) - fmt.Printf("empty CHILD shard root = %x\n", emptyChild.GetRootHashRaw()) - - realRoot := make([]byte, 32) - for i := range realRoot { - realRoot[i] = 0xAA - } - - p1 := NewParentSparseMerkleTree(api.SHA256, shardIDLen) - if err := p1.AddLeaf(big.NewInt(0b10000), realRoot); err != nil { - t.Fatal(err) - } - fmt.Printf("IR.h with shard 1111 silent = %x\n", p1.GetRootHashRaw()) - - p2 := NewParentSparseMerkleTree(api.SHA256, shardIDLen) - if err := p2.AddLeaf(big.NewInt(0b10000), realRoot); err != nil { - t.Fatal(err) - } - if err := p2.AddLeaf(big.NewInt(0b11111), emptyChild.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - fmt.Printf("IR.h with shard 1111 reporting 0 = %x\n", p2.GetRootHashRaw()) - - // dependence on SHARD_ID_LENGTH for the *empty* tree, which is certified - // verbatim by processRound when no shard reported this round. - for _, kl := range []int{1, 2, 3, 4, 8, 16} { - tr := NewParentSparseMerkleTree(api.SHA256, kl) - fmt.Printf("empty parent IR.h @ SHARD_ID_LENGTH=%2d -> %x\n", kl, tr.GetRootHashRaw()) - } -} - -// Cleanest canonicity counterexample: exactly one live shard. The -// whole-partition RSMT root then equals that shard's own root (all keys share -// the shard prefix, so the child tree IS the partition tree). -func TestAudit_SingleLiveShardRootIsNotChildRoot(t *testing.T) { - realRoot := make([]byte, 32) - for i := range realRoot { - realRoot[i] = 0xAA - } - for _, kl := range []int{1, 2, 4} { - p := NewParentSparseMerkleTree(api.SHA256, kl) - sid := big.NewInt(int64(1< key bits 0000 ; 0b10001 = 17 -> ? - sidA := big.NewInt(0b10000) - sidB := big.NewInt(0b11111) - - keyA, _ := api.PathToFixedBytes(sidA, keyLength) - keyB, _ := api.PathToFixedBytes(sidB, keyLength) - fmt.Printf("keyA=%x keyB=%x\n", keyA, keyB) - - if err := tree.AddLeaf(sidA, vA); err != nil { - t.Fatal(err) - } - if err := tree.AddLeaf(sidB, vB); err != nil { - t.Fatal(err) - } - parentRoot := tree.GetRootHashRaw() - fmt.Printf("PARENT-mode root (2 shards live, SHARD_ID_LENGTH=4) = %x\n", parentRoot) - - // canonical RSMT over the same spliced child-leaf hashes: - // keys 0000 and 1111 bifurcate at depth 0. - d := 0 - for ; d < keyLength; d++ { - if api.KeyBitBE(keyA, d) != api.KeyBitBE(keyB, d) { - break - } - } - fmt.Printf("bifurcation depth = %d\n", d) - var canon []byte - if api.KeyBitBE(keyA, d) == 0 { - canon = rsmtNode(d, keyA, vA, vB) - } else { - canon = rsmtNode(d, keyA, vB, vA) - } - fmt.Printf("CANONICAL RSMT root (splice semantics) = %x\n", canon) - - // also: standalone tree with the same two (path,value) pairs, ordinary leaves - std := NewSparseMerkleTree(api.SHA256, keyLength) - if err := std.AddLeaf(sidA, vA); err != nil { - t.Fatal(err) - } - if err := std.AddLeaf(sidB, vB); err != nil { - t.Fatal(err) - } - fmt.Printf("STANDALONE (rsmt_leaf_hash leaves) root = %x\n", std.GetRootHashRaw()) - - // same two leaves but parent mode with SHARD_ID_LENGTH=1 is not possible - // (only 2 slots); use keyLength=4 vs 5 to show dependence on the parameter. -} - -func TestAudit_FragmentAllZeroSibling(t *testing.T) { - const keyLength = 4 - tree := NewParentSparseMerkleTree(api.SHA256, keyLength) - vA := make([]byte, 32) - for i := range vA { - vA[i] = 0xAA - } - sidA := big.NewInt(0b10000) // key bits 0000 - if err := tree.AddLeaf(sidA, vA); err != nil { - t.Fatal(err) - } - vB := make([]byte, 32) - for i := range vB { - vB[i] = 0xBB - } - sidB := big.NewInt(0b11111) - if err := tree.AddLeaf(sidB, vB); err != nil { - t.Fatal(err) - } - - frag, err := tree.GetShardInclusionFragment(api.ShardID(0b10000)) - if err != nil { - t.Fatal(err) - } - if frag == nil { - t.Fatal("nil fragment") - } - var cert api.InclusionCert - if err := cert.UnmarshalBinary(frag.CertificateBytes); err != nil { - t.Fatal(err) - } - fmt.Printf("fragment shard leaf value = %x\n", frag.ShardLeafValue) - fmt.Printf("bitmap (first 4 bytes) = %x\n", cert.Bitmap[:4]) - depths := []int{} - for d := 0; d < 256; d++ { - if api.KeyBitBE(cert.Bitmap[:], d) == 1 { - depths = append(depths, d) - } - } - fmt.Printf("bitmap set depths = %v\n", depths) - zero := make([]byte, 32) - for i, s := range cert.Siblings { - isZero := string(s[:]) == string(zero) - fmt.Printf("sibling[%d] = %x allZero=%v\n", i, s[:], isZero) - } - - // verify the fragment against the parent root, i.e. confirm the phantom - // junctions are load-bearing in the certified path - root := tree.GetRootHashRaw() - keyA, _ := api.PathToFixedBytes(sidA, keyLength) - fullKey := make([]byte, 32) - copy(fullKey, keyA) - fmt.Printf("root=%x\n", root) - _ = fullKey -} diff --git a/internal/smt/zz_lens_parentcanon_test.go b/internal/smt/zz_lens_parentcanon_test.go deleted file mode 100644 index e1784311..00000000 --- a/internal/smt/zz_lens_parentcanon_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package smt - -import ( - "bytes" - "encoding/hex" - "math/big" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -func mkKey(firstByte byte, tail byte) []byte { - k := make([]byte, 32) - k[0] = firstByte - for i := 1; i < 32; i++ { - k[i] = tail - } - return k -} - -func mkVal(b byte) []byte { - v := make([]byte, 32) - for i := range v { - v[i] = b - } - return v -} - -func pathOf(t *testing.T, key []byte) *big.Int { - t.Helper() - p, err := api.FixedBytesToPath(key, api.StateTreeKeyLengthBits) - if err != nil { - t.Fatal(err) - } - return p -} - -func canonicalRoot(t *testing.T, kv map[string][]byte) []byte { - t.Helper() - std := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) - for ks, v := range kv { - key := []byte(ks) - if err := std.AddLeaf(pathOf(t, key), v); err != nil { - t.Fatal(err) - } - } - return std.GetRootHashRaw() -} - -// Case 1: fully-populated, every shard non-empty. Splice should be canonical. -func TestLens_ParentSplice_FullyOccupied(t *testing.T) { - keyA := mkKey(0x00, 0x11) // bit 0 = 0 -> shard 0b10 - keyB := mkKey(0x80, 0x22) // bit 0 = 1 -> shard 0b11 - valA, valB := mkVal(0xAA), mkVal(0xBB) - - c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b10) - if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { - t.Fatal(err) - } - c1 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b11) - if err := c1.AddLeaf(pathOf(t, keyB), valB); err != nil { - t.Fatal(err) - } - - parent := NewParentSparseMerkleTree(api.SHA256, 1) - if err := parent.AddLeaf(big.NewInt(0b10), c0.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - if err := parent.AddLeaf(big.NewInt(0b11), c1.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - - canon := canonicalRoot(t, map[string][]byte{string(keyA): valA, string(keyB): valB}) - got := parent.GetRootHashRaw() - t.Logf("canonical=%x parent=%x", canon, got) - if !bytes.Equal(canon, got) { - t.Errorf("expected match in fully-occupied case") - } -} - -// Case 2: SHARD_ID_LENGTH=1 (the reference sharding-compose.yml value), both -// shards live and configured, but shard 0b11 had no commitments this round so -// its child SMT root is the empty-tree root. -func TestLens_ParentSplice_LiveButEmptyShard(t *testing.T) { - keyA := mkKey(0x00, 0x11) - valA := mkVal(0xAA) - - c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b10) - if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { - t.Fatal(err) - } - c1 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b11) - emptyChildRoot := c1.GetRootHashRaw() - t.Logf("empty CHILD tree root = %x", emptyChildRoot) - - parent := NewParentSparseMerkleTree(api.SHA256, 1) - if err := parent.AddLeaf(big.NewInt(0b10), c0.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - if err := parent.AddLeaf(big.NewInt(0b11), emptyChildRoot); err != nil { - t.Fatal(err) - } - parentRoot := parent.GetRootHashRaw() - - canon := canonicalRoot(t, map[string][]byte{string(keyA): valA}) - t.Logf("canonical RSMT root of {A} = %x", canon) - t.Logf("certified parent root (IR.h) = %x", parentRoot) - if bytes.Equal(canon, parentRoot) { - t.Errorf("unexpectedly equal") - } - - // What ships to the end client. - frag, err := parent.GetShardInclusionFragment(0b10) - if err != nil { - t.Fatal(err) - } - var pcert api.InclusionCert - if err := pcert.UnmarshalBinary(frag.CertificateBytes); err != nil { - t.Fatal(err) - } - childCert, err := c0.GetInclusionCert(keyA) - if err != nil { - t.Fatal(err) - } - composed, err := api.ComposeInclusionCert(frag, childCert, c0.GetRootHashRaw()) - if err != nil { - t.Fatal(err) - } - depths := []int{} - for d := 0; d < 256; d++ { - if api.KeyBitBE(composed.Bitmap[:], d) == 1 { - depths = append(depths, d) - } - } - t.Logf("composed cert junction depths = %v", depths) - for i, s := range composed.Siblings { - t.Logf(" sibling[%d] = %x", i, s) - } - // Canonical: a 1-leaf RSMT has zero junctions, so the canonical cert is empty. - if err := composed.Verify(keyA, valA, parentRoot, api.SHA256); err != nil { - t.Errorf("composed cert should verify against the certified (non-canonical) root: %v", err) - } - if err := composed.Verify(keyA, valA, canon, api.SHA256); err == nil { - t.Errorf("composed cert unexpectedly verifies against the canonical root") - } -} - -// Case 3: SHARD_ID_LENGTH=4 default, only shards 0000 and 1111 live. -func TestLens_ParentSplice_PartiallyLive16(t *testing.T) { - keyA := mkKey(0x00, 0x11) // top 4 bits 0000 - keyB := mkKey(0xF0, 0x22) // top 4 bits 1111 - valA, valB := mkVal(0xAA), mkVal(0xBB) - - c0 := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b1_0000) - if err := c0.AddLeaf(pathOf(t, keyA), valA); err != nil { - t.Fatal(err) - } - cF := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, 0b1_1111) - if err := cF.AddLeaf(pathOf(t, keyB), valB); err != nil { - t.Fatal(err) - } - - parent := NewParentSparseMerkleTree(api.SHA256, 4) - t.Logf("EMPTY parent tree root (keyLength=4) = %x", parent.GetRootHashRaw()) - if err := parent.AddLeaf(big.NewInt(0b1_0000), c0.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - if err := parent.AddLeaf(big.NewInt(0b1_1111), cF.GetRootHashRaw()); err != nil { - t.Fatal(err) - } - parentRoot := parent.GetRootHashRaw() - canon := canonicalRoot(t, map[string][]byte{string(keyA): valA, string(keyB): valB}) - t.Logf("canonical RSMT root = %x", canon) - t.Logf("parent (certified) = %x", parentRoot) - if bytes.Equal(canon, parentRoot) { - t.Errorf("unexpectedly equal") - } - - frag, err := parent.GetShardInclusionFragment(0b1_0000) - if err != nil { - t.Fatal(err) - } - childCert, err := c0.GetInclusionCert(keyA) - if err != nil { - t.Fatal(err) - } - composed, err := api.ComposeInclusionCert(frag, childCert, c0.GetRootHashRaw()) - if err != nil { - t.Fatal(err) - } - depths := []int{} - for d := 0; d < 256; d++ { - if api.KeyBitBE(composed.Bitmap[:], d) == 1 { - depths = append(depths, d) - } - } - t.Logf("composed cert junction depths = %v (canonical would be [0])", depths) - zeros := 0 - for i, s := range composed.Siblings { - t.Logf(" sibling[%d] = %s", i, hex.EncodeToString(s[:])) - if bytes.Equal(s[:], make([]byte, 32)) { - zeros++ - } - } - t.Logf("all-zero siblings shipped to client: %d", zeros) - if err := composed.Verify(keyA, valA, parentRoot, api.SHA256); err != nil { - t.Errorf("composed cert must verify against certified root: %v", err) - } -} - -// Case 4: empty-tree root, standalone vs parent mode. -func TestLens_EmptyTreeRoots(t *testing.T) { - t.Logf("empty standalone (256-bit) root = %x", NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits).GetRootHashRaw()) - for _, n := range []int{1, 2, 4, 8} { - t.Logf("empty parent root SHARD_ID_LENGTH=%2d = %x", n, NewParentSparseMerkleTree(api.SHA256, n).GetRootHashRaw()) - } -} diff --git a/internal/smt/zz_specaudit2_test.go b/internal/smt/zz_specaudit2_test.go deleted file mode 100644 index eb123d71..00000000 --- a/internal/smt/zz_specaudit2_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package smt - -import ( - "crypto/rand" - "math/big" - mrand "math/rand" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// Differential fuzz: random (possibly nonsense) certs, keys, values, roots. -// The impl verifier and the literal spec verifier must always agree. -func TestSpec_DifferentialFuzz(t *testing.T) { - rng := mrand.New(mrand.NewSource(0xC0FFEE)) - for iter := 0; iter < 20000; iter++ { - var bm [32]byte - nbits := rng.Intn(6) - for i := 0; i < nbits; i++ { - api.SetBitBE(bm[:], rng.Intn(256)) - } - pc := 0 - for d := 0; d < 256; d++ { - pc += int(api.KeyBitBE(bm[:], d)) - } - // sometimes deliberately mismatch the sibling count - n := pc - switch rng.Intn(4) { - case 0: - n = pc + 1 - case 1: - if pc > 0 { - n = pc - 1 - } - } - sibs := make([][32]byte, n) - for i := range sibs { - rand.Read(sibs[i][:]) - } - k := make([]byte, 32) - v := make([]byte, rng.Intn(40)) - root := make([]byte, 32) - rand.Read(k) - rand.Read(v) - rand.Read(root) - - cert := &api.InclusionCert{Bitmap: bm, Siblings: sibs} - implOK := cert.Verify(k, v, root, api.SHA256) == nil - - specSibs := make([][]byte, len(sibs)) - for i := range sibs { - specSibs[i] = sibs[i][:] - } - specOK := specVerifyInclusion(bm[:], specSibs, root, k, v) - if implOK != specOK { - t.Fatalf("iter %d DIVERGENCE: impl=%v spec=%v bitmap=%x nsibs=%d", iter, implOK, specOK, bm, n) - } - } -} - -// Forgery attempt: take a genuine cert for leaf A, and try to make it verify -// for a different key that shares the path structure. Spec forbids it because -// regions are derived from the queried key. -func TestSpec_RegionBindsQueriedKey(t *testing.T) { - tree, leaves := buildRandomTree(t, 128) - root := tree.GetRootHashRaw() - for _, l := range leaves[:8] { - cert, err := tree.GetInclusionCert(l.k) - if err != nil { - t.Fatal(err) - } - // flip a key bit at a depth BELOW the deepest junction: descent side - // unchanged, region changes only at depths > flipped bit. - deepest := -1 - for d := 0; d < 256; d++ { - if api.KeyBitBE(cert.Bitmap[:], d) == 1 { - deepest = d - } - } - if deepest >= 255 { - continue - } - bad := append([]byte(nil), l.k...) - bad[(deepest+1)/8] ^= 0x80 >> uint((deepest+1)%8) - if err := cert.Verify(bad, l.v, root, api.SHA256); err == nil { - t.Fatalf("impl ACCEPTED cert for a key differing below deepest junction (deepest=%d)", deepest) - } - } -} - -// Sharded composition: does the certificate handed to a client verify under -// the LITERAL spec verifier with the state id as key? -func TestSpec_ComposedShardCertUnderSpec(t *testing.T) { - const shardBits = 4 - parent := NewParentSparseMerkleTree(api.SHA256, shardBits) - - // shard 0b1010 -> shardID sentinel path 0b1_1010? Shard id encoding: - // sentinel-prefixed int, BitLen()-1 == shardBits. - shardID := api.ShardID(0b1_0110) // shard bits (path bit order) = 0,1,1,0 - child := NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, shardID) - - // build keys that route to this shard - var kvs []kv - for len(kvs) < 5 { - k := make([]byte, 32) - rand.Read(k) - p, err := api.FixedBytesToPath(k, api.StateTreeKeyLengthBits) - if err != nil { - t.Fatal(err) - } - v := make([]byte, 32) - rand.Read(v) - if err := child.AddLeaf(p, v); err != nil { - continue // wrong shard - } - kvs = append(kvs, kv{k, v}) - } - childRoot := child.GetRootHashRaw() - - // publish child root into the parent tree - shardPath := big.NewInt(int64(shardID)) - if err := parent.AddLeaf(shardPath, childRoot); err != nil { - t.Fatal(err) - } - // fill the other shards with random roots so siblings are non-trivial - for s := 0; s < 1<= 0; d-- { - if specBit(bitmap, d) == 0 { - continue - } - j-- - s := siblings[j] - p := specRegion(k, d) - var hL, hR []byte - if specBit(k, d) == 1 { - hL, hR = s, h - } else { - hL, hR = h, s - } - h = specNodeHash(hL, hR, d, p) - } - return h -} - -// Spec-constructed certificates (arbitrary junction depth sets, including 0 -// and 255, arbitrary value lengths) must be accepted by the implementation -// verifier and survive a wire round-trip. -func TestSpec_ImplAcceptsSpecConstructedCerts(t *testing.T) { - rng := mrand.New(mrand.NewSource(42)) - for iter := 0; iter < 5000; iter++ { - var bm [32]byte - depths := map[int]bool{} - nd := rng.Intn(8) - if iter < 3 { - // force the extreme depths at least once each - depths[0] = true - depths[255] = true - } - for i := 0; i < nd; i++ { - depths[rng.Intn(256)] = true - } - ordered := make([]int, 0, len(depths)) - for d := 0; d < 256; d++ { - if depths[d] { - api.SetBitBE(bm[:], d) - ordered = append(ordered, d) - } - } - sibs := make([][32]byte, len(ordered)) - for i := range sibs { - rand.Read(sibs[i][:]) - } - k := make([]byte, 32) - rand.Read(k) - v := make([]byte, rng.Intn(64)) - rand.Read(v) - - specSibs := make([][]byte, len(sibs)) - for i := range sibs { - specSibs[i] = sibs[i][:] - } - root := specRoot(bm[:], specSibs, k, v) - - cert := &api.InclusionCert{Bitmap: bm, Siblings: sibs} - if err := cert.Verify(k, v, root, api.SHA256); err != nil { - t.Fatalf("iter %d: impl REJECTED a spec-valid cert (depths=%v): %v", iter, ordered, err) - } - - // wire round trip: bitmap[32] || s_1..s_n, total 32+32n bytes - wire, err := cert.MarshalBinary() - if err != nil { - t.Fatal(err) - } - if len(wire) != 32+32*len(sibs) { - t.Fatalf("iter %d: wire length %d, want %d", iter, len(wire), 32+32*len(sibs)) - } - if !bytes.Equal(wire[:32], bm[:]) { - t.Fatalf("iter %d: bitmap is not the first 32 wire bytes", iter) - } - var back api.InclusionCert - if err := back.UnmarshalBinary(wire); err != nil { - t.Fatalf("iter %d: round trip decode: %v", iter, err) - } - if err := back.Verify(k, v, root, api.SHA256); err != nil { - t.Fatalf("iter %d: decoded cert rejected: %v", iter, err) - } - - // trailing garbage / truncation must be rejected - if err := (&api.InclusionCert{}).UnmarshalBinary(append(append([]byte(nil), wire...), 0x00)); err == nil { - t.Fatalf("iter %d: decoder accepted 1 trailing byte", iter) - } - if err := (&api.InclusionCert{}).UnmarshalBinary(append(append([]byte(nil), wire...), make([]byte, 32)...)); err == nil { - t.Fatalf("iter %d: decoder accepted an extra 32-byte sibling", iter) - } - if err := (&api.InclusionCert{}).UnmarshalBinary(wire[:len(wire)-1]); err == nil { - t.Fatalf("iter %d: decoder accepted a truncated cert", iter) - } - } -} - -// A junction at depth 255 (keys differing only in the final bit) must be -// generated and verified correctly end to end. -func TestSpec_Depth255Junction(t *testing.T) { - tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) - kA := bytes.Repeat([]byte{0xAB}, 32) - kB := append([]byte(nil), kA...) - kB[31] ^= 0x01 // differ only at bit 255 - vA, vB := []byte("A"), []byte("B") - pA, _ := api.FixedBytesToPath(kA, 256) - pB, _ := api.FixedBytesToPath(kB, 256) - if err := tree.AddLeaf(pA, vA); err != nil { - t.Fatal(err) - } - if err := tree.AddLeaf(pB, vB); err != nil { - t.Fatal(err) - } - root := tree.GetRootHashRaw() - cert, err := tree.GetInclusionCert(kA) - if err != nil { - t.Fatal(err) - } - if api.KeyBitBE(cert.Bitmap[:], 255) != 1 || len(cert.Siblings) != 1 { - t.Fatalf("expected a single junction at depth 255, bitmap=%x n=%d", cert.Bitmap, len(cert.Siblings)) - } - bm, sibs := certToSpec(cert) - if !specVerifyInclusion(bm, sibs, root, kA, vA) { - t.Fatal("spec verifier rejected depth-255 cert") - } - // kA has bit 255 = 1 (0xAB) so kA is the right child, kB (0xAA) the left. - want := specNodeHash(specLeafHash(kB, vB), specLeafHash(kA, vA), 255, specRegion(kA, 255)) - if !bytes.Equal(root, want) { - t.Fatalf("root mismatch at depth 255:\n impl %x\n spec %x", root, want) - } -} diff --git a/internal/smt/zz_specaudit4_test.go b/internal/smt/zz_specaudit4_test.go deleted file mode 100644 index 0d311119..00000000 --- a/internal/smt/zz_specaudit4_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package smt - -import ( - "crypto/rand" - "math/big" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// Probe: with only two of 16 shards populated, the canonical RSMT has exactly -// one junction (depth 0) on the path to either shard. Does the parent tree -// emit a compressed path, or does it commit phantom all-zero subtrees? -func TestProbe_ParentTreePhantomSubtrees(t *testing.T) { - const shardBits = 4 - parent := NewParentSparseMerkleTree(api.SHA256, shardBits) - - rA := make([]byte, 32) - rB := make([]byte, 32) - rand.Read(rA) - rand.Read(rB) - - // shard path bits 0..3 = 0,0,0,0 -> sentinel int 0b1_0000 = 16 - // shard path bits 0..3 = 1,1,1,1 -> sentinel int 0b1_1111 = 31 - if err := parent.AddLeaf(big.NewInt(16), rA); err != nil { - t.Fatal(err) - } - if err := parent.AddLeaf(big.NewInt(31), rB); err != nil { - t.Fatal(err) - } - - frag, err := parent.GetShardInclusionFragment(16) - if err != nil { - t.Fatal(err) - } - var cert api.InclusionCert - if err := cert.UnmarshalBinary(frag.CertificateBytes); err != nil { - t.Fatal(err) - } - depths := []int{} - for d := 0; d < 256; d++ { - if api.KeyBitBE(cert.Bitmap[:], d) == 1 { - depths = append(depths, d) - } - } - t.Logf("junction depths on path to shard 0000 with only 2/16 shards live: %v (siblings=%d)", depths, len(cert.Siblings)) - for i, s := range cert.Siblings { - t.Logf(" sibling[%d] = %x", i, s) - } - t.Logf("parent root = %x", parent.GetRootHashRaw()) -} diff --git a/internal/smt/zz_specaudit_test.go b/internal/smt/zz_specaudit_test.go deleted file mode 100644 index 89c5c8c3..00000000 --- a/internal/smt/zz_specaudit_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package smt - -import ( - "bytes" - "crypto/rand" - "crypto/sha256" - "testing" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// --------------------------------------------------------------------------- -// Literal transcription of appendix-hashtrees.tex Sec. C.3.7.1 -// (rsmt_verify_inclusion) + C.3.2.1/C.3.2.2 (leaf/node hash) + C.3.6 (cert -// path bit numbering). Written from the paper only, no reuse of pkg/api. -// --------------------------------------------------------------------------- - -// bit d of a big-endian bit string: k[0] is MSB of byte 0. -func specBit(buf []byte, d int) byte { return (buf[d/8] >> (7 - uint(d)%8)) & 1 } - -func specPopcount(bm []byte) int { - n := 0 - for d := 0; d < 8*len(bm); d++ { - n += int(specBit(bm, d)) - } - return n -} - -//

for p = k[0..d): 256-bit BE string, p in the first d bits, rest zero. -func specRegion(k []byte, d int) []byte { - p := make([]byte, 32) - for i := 0; i < d; i++ { - if specBit(k, i) == 1 { - p[i/8] |= 0x80 >> (uint(i) % 8) - } - } - return p -} - -func specLeafHash(k, v []byte) []byte { - h := sha256.New() - h.Write([]byte{0x00}) - h.Write(k) - h.Write(v) - return h.Sum(nil) -} - -func specNodeHash(hL, hR []byte, d int, p []byte) []byte { - h := sha256.New() - h.Write([]byte{0x01}) - h.Write([]byte{byte(d)}) - h.Write(p) - h.Write(hL) - h.Write(hR) - return h.Sum(nil) -} - -func specVerifyInclusion(bitmap []byte, siblings [][]byte, rho, k, v []byte) bool { - if len(bitmap) != 32 || len(k) != 32 || len(rho) != 32 { - return false - } - if len(siblings) != specPopcount(bitmap) { - return false - } - h := specLeafHash(k, v) - j := len(siblings) - for d := 255; d >= 0; d-- { - if specBit(bitmap, d) == 0 { - continue - } - j-- - s := siblings[j] - if len(s) != 32 { - return false - } - p := specRegion(k, d) - var hL, hR []byte - if specBit(k, d) == 1 { - hL, hR = s, h - } else { - hL, hR = h, s - } - h = specNodeHash(hL, hR, d, p) - } - return j == 0 && bytes.Equal(h, rho) -} - -func certToSpec(c *api.InclusionCert) ([]byte, [][]byte) { - sibs := make([][]byte, len(c.Siblings)) - for i := range c.Siblings { - sibs[i] = append([]byte(nil), c.Siblings[i][:]...) - } - return append([]byte(nil), c.Bitmap[:]...), sibs -} - -// --------------------------------------------------------------------------- - -type kv struct{ k, v []byte } - -func buildRandomTree(t *testing.T, n int) (*SparseMerkleTree, []kv) { - t.Helper() - tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) - out := make([]kv, 0, n) - seen := map[string]bool{} - for len(out) < n { - k := make([]byte, 32) - if _, err := rand.Read(k); err != nil { - t.Fatal(err) - } - if seen[string(k)] { - continue - } - seen[string(k)] = true - v := make([]byte, 32) - if _, err := rand.Read(v); err != nil { - t.Fatal(err) - } - p, err := api.FixedBytesToPath(k, api.StateTreeKeyLengthBits) - if err != nil { - t.Fatal(err) - } - if err := tree.AddLeaf(p, v); err != nil { - t.Fatal(err) - } - out = append(out, kv{k, v}) - } - return tree, out -} - -// TestSpec_GeneratedCertsVerifyUnderSpec: every cert the implementation -// generates must be accepted by the literal spec verifier. -func TestSpec_GeneratedCertsVerifyUnderSpec(t *testing.T) { - for _, n := range []int{1, 2, 3, 7, 33, 200} { - tree, leaves := buildRandomTree(t, n) - root := tree.GetRootHashRaw() - for _, l := range leaves { - cert, err := tree.GetInclusionCert(l.k) - if err != nil { - t.Fatalf("n=%d GetInclusionCert: %v", n, err) - } - bm, sibs := certToSpec(cert) - if !specVerifyInclusion(bm, sibs, root, l.k, l.v) { - t.Fatalf("n=%d: spec verifier REJECTED an implementation-generated cert for key %x", n, l.k) - } - if err := cert.Verify(l.k, l.v, root, api.SHA256); err != nil { - t.Fatalf("n=%d: impl verifier rejected own cert: %v", n, err) - } - } - } -} - -// TestSpec_ImplAgreesWithSpecOnMutations: for a corpus of mutated certs the -// impl verifier and the spec verifier must give the same answer. -func TestSpec_ImplAgreesWithSpecOnMutations(t *testing.T) { - tree, leaves := buildRandomTree(t, 64) - root := tree.GetRootHashRaw() - - type tc struct { - name string - mut func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) - } - cases := []tc{ - {"unchanged", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { return c, k, v }}, - {"extra sibling appended (no bitmap bit)", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - d := *c - d.Siblings = append(append([][32]byte{}, c.Siblings...), [32]byte{}) - return &d, k, v - }}, - {"sibling dropped (bitmap unchanged)", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - if len(c.Siblings) == 0 { - return c, k, v - } - d := *c - d.Siblings = append([][32]byte{}, c.Siblings[:len(c.Siblings)-1]...) - return &d, k, v - }}, - {"bitmap bit flipped on at unused depth", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - d := *c - for depth := 255; depth >= 0; depth-- { - if api.KeyBitBE(d.Bitmap[:], depth) == 0 { - api.SetBitBE(d.Bitmap[:], depth) - break - } - } - return &d, k, v - }}, - {"siblings reversed", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - d := *c - s := append([][32]byte{}, c.Siblings...) - for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { - s[i], s[j] = s[j], s[i] - } - d.Siblings = s - return &d, k, v - }}, - {"wrong value", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - w := append([]byte(nil), v...) - w[0] ^= 0xff - return c, k, w - }}, - {"wrong key", func(c *api.InclusionCert, k, v []byte) (*api.InclusionCert, []byte, []byte) { - w := append([]byte(nil), k...) - w[31] ^= 0x01 - return c, w, v - }}, - } - - for _, l := range leaves[:16] { - base, err := tree.GetInclusionCert(l.k) - if err != nil { - t.Fatal(err) - } - for _, c := range cases { - mc, k, v := c.mut(base, l.k, l.v) - bm, sibs := certToSpec(mc) - specOK := specVerifyInclusion(bm, sibs, root, k, v) - implOK := mc.Verify(k, v, root, api.SHA256) == nil - if specOK != implOK { - t.Errorf("DIVERGENCE [%s] key=%x: spec=%v impl=%v", c.name, l.k, specOK, implOK) - } - } - } -} - -// TestSpec_RootHashMatchesSpecReconstruction: recompute the tree root purely -// from the spec (leaf hashes + certificate paths) for a 2-leaf tree. -func TestSpec_TwoLeafRootFromSpec(t *testing.T) { - tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) - kA := make([]byte, 32) - kB := make([]byte, 32) - kA[0] = 0x00 // bit0=0 - kB[0] = 0x80 // bit0=1 - vA := []byte("a") - vB := []byte("b") - pA, _ := api.FixedBytesToPath(kA, 256) - pB, _ := api.FixedBytesToPath(kB, 256) - if err := tree.AddLeaf(pA, vA); err != nil { - t.Fatal(err) - } - if err := tree.AddLeaf(pB, vB); err != nil { - t.Fatal(err) - } - root := tree.GetRootHashRaw() - // spec: junction at depth 0, region = empty (all-zero 32 bytes) - want := specNodeHash(specLeafHash(kA, vA), specLeafHash(kB, vB), 0, make([]byte, 32)) - if !bytes.Equal(root, want) { - t.Fatalf("root mismatch\n impl %x\n spec %x", root, want) - } - certA, err := tree.GetInclusionCert(kA) - if err != nil { - t.Fatal(err) - } - t.Logf("certA bitmap=%x siblings=%d", certA.Bitmap, len(certA.Siblings)) -} diff --git a/pkg/api/zz_audit_networkid_test.go b/pkg/api/zz_audit_networkid_test.go deleted file mode 100644 index 5c568fb6..00000000 --- a/pkg/api/zz_audit_networkid_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package api - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/unicitynetwork/bft-go-base/types" -) - -// AUDIT: platform.tex:456 requires ensure(UC.C^r.alpha = T.alpha). The seal here -// claims NetworkTestNet while the trust base is NetworkMainNet; the same root -// key signs both, so only an explicit network-ID comparison can catch it. -func TestAudit_NetworkIDNotChecked(t *testing.T) { - orig := auditSealNetworkID - auditSealNetworkID = types.NetworkTestNet - defer func() { auditSealNetworkID = orig }() - - proof, req, partitionID, tb, sid0, _ := buildProofCommittedInShard( - t, "1111111111111111111111111111111111111111111111111111111111111111", 0) - require.Equal(t, types.NetworkMainNet, tb.GetNetworkID()) - - var uc types.UnicityCertificate - require.NoError(t, types.Cbor.Unmarshal(proof.UnicityCertificate, &uc)) - t.Logf("seal.NetworkID=%d trustbase.NetworkID=%d", uc.UnicitySeal.NetworkID, tb.GetNetworkID()) - - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: partitionID, ExpectedShardID: sid0, - }) - t.Logf("Verify() returned: %v", err) - require.NoError(t, err, "AUDIT: cross-network UC accepted") -} diff --git a/pkg/api/zz_audit_shardbinding_test.go b/pkg/api/zz_audit_shardbinding_test.go deleted file mode 100644 index bb2bf8de..00000000 --- a/pkg/api/zz_audit_shardbinding_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package api - -import ( - "crypto" - "testing" - - "github.com/stretchr/testify/require" - test "github.com/unicitynetwork/bft-go-base/testutils" - testsig "github.com/unicitynetwork/bft-go-base/testutils/sig" - "github.com/unicitynetwork/bft-go-base/types" -) - -// auditSealNetworkID is the network ID stamped into the UnicitySeal; the trust -// base is always built for types.NetworkMainNet. -var auditSealNetworkID = types.NetworkMainNet - -// buildProofCommittedInShard is buildSignedSingleLeafProof, except the caller -// chooses which shard's InputRecord actually carries the single-leaf SMT root. -// That lets us commit a key whose canonical shard is sid0 inside sid1's tree. -func buildProofCommittedInShard(t *testing.T, stateIDHex string, committedIn int) ( - *InclusionProofV2, *CertificationRequest, types.PartitionID, types.RootTrustBase, - types.ShardID, types.ShardID, -) { - t.Helper() - - stateID := RequireNewImprintV2(stateIDHex) - txHash := RequireNewImprintV2("2222222222222222222222222222222222222222222222222222222222222222") - - req := &CertificationRequest{ - StateID: stateID, - CertificationData: CertificationData{TransactionHash: txHash}, - } - - key, err := stateID.GetTreeKey() - require.NoError(t, err) - const referenceTime uint64 = 1755000000 - hasher := NewDataHasher(InclusionProofV2HashAlgorithm) - hasher.Reset(). - AddData([]byte{0x00}). - AddData(key). - AddData(LeafValue(txHash.DataBytes(), referenceTime)) - leafRoot := append([]byte(nil), hasher.GetHash().RawHash...) - - cert := &InclusionCert{} - certBytes, err := cert.MarshalBinary() - require.NoError(t, err) - - sid0, sid1 := types.ShardID{}.Split() - const partitionID types.PartitionID = 0x0f0f0f0f - - mkIR := func(h []byte, tag byte) *types.InputRecord { - return &types.InputRecord{ - Version: 1, PreviousHash: []byte{0, 0, tag}, Hash: h, - BlockHash: []byte{0, 0, tag + 1}, SummaryValue: []byte{0, 0, tag + 2}, - Timestamp: types.NewTimestamp(), RoundNumber: 1, - } - } - var ir0, ir1 *types.InputRecord - if committedIn == 0 { - ir0, ir1 = mkIR(leafRoot, 1), mkIR(test.RandomBytes(32), 5) - } else { - ir0, ir1 = mkIR(test.RandomBytes(32), 1), mkIR(leafRoot, 5) - } - trHash0 := test.RandomBytes(32) - trHash1 := test.RandomBytes(32) - - sTree, err := types.CreateShardTree( - types.ShardingScheme{sid0, sid1}, - []types.ShardTreeInput{ - {Shard: sid0, IR: ir0, TRHash: trHash0}, - {Shard: sid1, IR: ir1, TRHash: trHash1}, - }, crypto.SHA256) - require.NoError(t, err) - - ownerShard, ownerIR, ownerTR := sid0, ir0, trHash0 - if committedIn == 1 { - ownerShard, ownerIR, ownerTR = sid1, ir1, trHash1 - } - stCert, err := sTree.Certificate(ownerShard) - require.NoError(t, err) - - ut, err := types.NewUnicityTree(crypto.SHA256, []*types.UnicityTreeData{{ - Partition: partitionID, ShardTreeRoot: sTree.RootHash(), - }}) - require.NoError(t, err) - utCert, err := ut.Certificate(partitionID) - require.NoError(t, err) - - signer, verifier := testsig.CreateSignerAndVerifier(t) - sigKey, err := verifier.MarshalPublicKey() - require.NoError(t, err) - tb, err := types.NewTrustBase(types.NetworkMainNet, []*types.NodeInfo{ - {NodeID: "test", SigKey: sigKey, Stake: 1}, - }) - require.NoError(t, err) - - seal := &types.UnicitySeal{ - Version: 1, NetworkID: auditSealNetworkID, RootChainRoundNumber: 1, - Timestamp: types.NewTimestamp(), - PreviousHash: test.RandomBytes(32), Hash: ut.RootHash(), - } - require.NoError(t, seal.Sign("test", signer)) - - ucBytes, err := types.Cbor.Marshal(types.UnicityCertificate{ - Version: 1, InputRecord: ownerIR, TRHash: ownerTR, - ShardTreeCertificate: stCert, UnicityTreeCertificate: utCert, UnicitySeal: seal, - }) - require.NoError(t, err) - - certifiedAt := referenceTime - proof := &InclusionProofV2{ - CertificationData: &req.CertificationData, - ReferenceTime: &certifiedAt, - CertificateBytes: certBytes, - UnicityCertificate: ucBytes, - } - return proof, req, partitionID, tb, sid0, sid1 -} - -// AUDIT: state ID 0x11.. has MSB 0, so f_SH(sid) = shard "0". The leaf is -// nevertheless committed in shard "1"'s SMT and certified by shard "1"'s UC. -// Per platform.tex:459 VerifyInclusionProof MUST reject. Go accepts. -func TestAudit_ForeignShardKeyAccepted(t *testing.T) { - proof, req, partitionID, tb, sid0, sid1 := buildProofCommittedInShard( - t, "1111111111111111111111111111111111111111111111111111111111111111", 1) - - key, err := req.StateID.GetTreeKey() - require.NoError(t, err) - t.Logf("key[0]=%08b f_SH(sid)=%q (sid0=%q, sid1=%q)", key[0], sid0.String(), sid0.String(), sid1.String()) - require.True(t, sid0.Comparator()(key), "key must route to shard 0") - require.False(t, sid1.Comparator()(key), "key must NOT route to shard 1") - - err = proof.Verify(req, &VerifierContext{ - TrustBase: tb, - PartitionID: partitionID, - ExpectedShardID: sid1, // the shard that served the proof - }) - t.Logf("Verify() returned: %v", err) - require.NoError(t, err, "AUDIT: Go verifier accepted a foreign-shard key") -} - -// AUDIT: the mandatory H(CD_beta) = UC.C^uni.dhash binding is skippable. -func TestAudit_ShardConfHashOptional(t *testing.T) { - proof, req, partitionID, tb, sid0, _ := buildProofCommittedInShard( - t, "1111111111111111111111111111111111111111111111111111111111111111", 0) - - // UC.ShardConfHash was never set (nil) yet verification passes when the - // verifier context leaves ShardConfHash nil. - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, - PartitionID: partitionID, - ExpectedShardID: sid0, - ShardConfHash: nil, - }) - t.Logf("Verify() with nil ShardConfHash returned: %v", err) - require.NoError(t, err) -} diff --git a/pkg/api/zz_xcheck_shardbind_test.go b/pkg/api/zz_xcheck_shardbind_test.go deleted file mode 100644 index acfd3c7f..00000000 --- a/pkg/api/zz_xcheck_shardbind_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package api - -import ( - "crypto" - "testing" - - "github.com/stretchr/testify/require" - test "github.com/unicitynetwork/bft-go-base/testutils" - testsig "github.com/unicitynetwork/bft-go-base/testutils/sig" - "github.com/unicitynetwork/bft-go-base/types" -) - -// xcheckBuild builds a fully signed v2 inclusion proof for a two-shard -// partition (SH = {"0","1"}) in which the leaf for `stateIDHex` is committed -// into `committingShard`'s SMT -- regardless of which shard f_SH(sid) actually -// names. netID is written into the seal; tbNet into the trust base. -// -// Independent of buildSignedSingleLeafProof in inclusion_proof_v2_verify_test.go: -// that helper always puts the leaf root in shard 0's IR, so it cannot express a -// foreign-shard commitment. -func xcheckBuild(t *testing.T, stateIDHex string, committingShard types.ShardID, - netID types.NetworkID, tbNet types.NetworkID) ( - *InclusionProofV2, *CertificationRequest, types.PartitionID, types.RootTrustBase) { - t.Helper() - - stateID := RequireNewImprintV2(stateIDHex) - txHash := RequireNewImprintV2("2222222222222222222222222222222222222222222222222222222222222222") - req := &CertificationRequest{ - StateID: stateID, - CertificationData: CertificationData{TransactionHash: txHash}, - } - - key, err := stateID.GetTreeKey() - require.NoError(t, err) - const referenceTime uint64 = 1755000000 - - // Single-leaf RSMT root: H(0x00 || key || v). - h := NewDataHasher(InclusionProofV2HashAlgorithm) - h.Reset(). - AddData([]byte{0x00}). - AddData(key). - AddData(LeafValue(txHash.DataBytes(), referenceTime)) - leafRoot := append([]byte(nil), h.GetHash().RawHash...) - - cert := &InclusionCert{} // single leaf, no siblings - certBytes, err := cert.MarshalBinary() - require.NoError(t, err) - - sid0, sid1 := types.ShardID{}.Split() - const partitionID types.PartitionID = 0x0f0f0f0f - - mkIR := func(hash []byte, salt byte) *types.InputRecord { - return &types.InputRecord{ - Version: 1, PreviousHash: []byte{0, 0, salt}, Hash: hash, - BlockHash: []byte{0, 0, salt + 1}, SummaryValue: []byte{0, 0, salt + 2}, - Timestamp: types.NewTimestamp(), RoundNumber: 1, Epoch: 0, - } - } - // The committing shard's IR carries the leaf root; the other shard is filler. - ir0, ir1 := mkIR(test.RandomBytes(32), 1), mkIR(test.RandomBytes(32), 5) - if committingShard.Equal(sid0) { - ir0 = mkIR(leafRoot, 1) - } else { - ir1 = mkIR(leafRoot, 5) - } - trHash0, trHash1 := test.RandomBytes(32), test.RandomBytes(32) - - sTree, err := types.CreateShardTree( - types.ShardingScheme{sid0, sid1}, - []types.ShardTreeInput{ - {Shard: sid0, IR: ir0, TRHash: trHash0}, - {Shard: sid1, IR: ir1, TRHash: trHash1}, - }, crypto.SHA256) - require.NoError(t, err) - - ownerIR, ownerTR := ir0, trHash0 - if committingShard.Equal(sid1) { - ownerIR, ownerTR = ir1, trHash1 - } - stCert, err := sTree.Certificate(committingShard) - require.NoError(t, err) - - ut, err := types.NewUnicityTree(crypto.SHA256, []*types.UnicityTreeData{ - {Partition: partitionID, ShardTreeRoot: sTree.RootHash()}, - }) - require.NoError(t, err) - utCert, err := ut.Certificate(partitionID) - require.NoError(t, err) - - signer, verifier := testsig.CreateSignerAndVerifier(t) - sigKey, err := verifier.MarshalPublicKey() - require.NoError(t, err) - tb, err := types.NewTrustBase(tbNet, []*types.NodeInfo{{NodeID: "n1", SigKey: sigKey, Stake: 1}}) - require.NoError(t, err) - - seal := &types.UnicitySeal{ - Version: 1, NetworkID: netID, RootChainRoundNumber: 1, - Timestamp: types.NewTimestamp(), PreviousHash: test.RandomBytes(32), - Hash: ut.RootHash(), - } - require.NoError(t, seal.Sign("n1", signer)) - - ucBytes, err := types.Cbor.Marshal(types.UnicityCertificate{ - Version: 1, InputRecord: ownerIR, TRHash: ownerTR, - ShardTreeCertificate: stCert, UnicityTreeCertificate: utCert, UnicitySeal: seal, - }) - require.NoError(t, err) - - rt := referenceTime - return &InclusionProofV2{ - CertificationData: &req.CertificationData, - ReferenceTime: &rt, - CertificateBytes: certBytes, - UnicityCertificate: ucBytes, - }, req, partitionID, tb -} - -// Establish the routing fact first: sid 0x1111... routes to shard "0". -func TestXCheck_RoutingFact(t *testing.T) { - sid0, sid1 := types.ShardID{}.Split() - key, err := RequireNewImprintV2( - "1111111111111111111111111111111111111111111111111111111111111111").GetTreeKey() - require.NoError(t, err) - require.True(t, sid0.Comparator()(key), "f_SH(sid) must be shard 0") - require.False(t, sid1.Comparator()(key), "sid must NOT route to shard 1") - t.Logf("key[0]=%#02x -> f_SH = shard %q (not %q)", key[0], sid0, sid1) -} - -// THE CLAIM: a leaf whose f_SH(sid) = "0" but which shard "1" committed and -// certified under a fully valid UC is accepted by InclusionProofV2.Verify. -func TestXCheck_ForeignShardLeafAccepted(t *testing.T) { - sid0, sid1 := types.ShardID{}.Split() - proof, req, pid, tb := xcheckBuild(t, - "1111111111111111111111111111111111111111111111111111111111111111", // f_SH = sid0 - sid1, // but committed + certified by shard 1 - types.NetworkMainNet, types.NetworkMainNet) - - // Caller derives ExpectedShardID from the serving endpoint / the proof's own - // UC -- the in-repo pattern -- i.e. sid1. - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: pid, ExpectedShardID: sid1, - }) - t.Logf("Verify(ExpectedShardID=sid1) for a sid0-routed key => %v", err) - require.NoError(t, err, "spec ensure(f_SH(sid)=sigma) would have rejected this") - - // Control: same construction with the key committed in its correct shard. - p2, r2, pid2, tb2 := xcheckBuild(t, - "1111111111111111111111111111111111111111111111111111111111111111", - sid0, types.NetworkMainNet, types.NetworkMainNet) - require.NoError(t, p2.Verify(r2, &VerifierContext{ - TrustBase: tb2, PartitionID: pid2, ExpectedShardID: sid0, - })) -} - -// Control the other way: the ONLY shard check is equality with the caller's -// value, so passing the true responsible shard rejects the honest proof too. -func TestXCheck_OnlyCheckIsCallerEquality(t *testing.T) { - sid0, sid1 := types.ShardID{}.Split() - proof, req, pid, tb := xcheckBuild(t, - "1111111111111111111111111111111111111111111111111111111111111111", - sid1, types.NetworkMainNet, types.NetworkMainNet) - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, - }) - require.ErrorContains(t, err, "invalid shard ID") - t.Logf("ExpectedShardID=sid0 => %v (equality with caller value, not f_SH)", err) -} - -// UC.C^r.alpha = T.alpha is unchecked: a testnet-sealed UC verifies against a -// mainnet trust base. -func TestXCheck_NetworkIDUnchecked(t *testing.T) { - sid0, _ := types.ShardID{}.Split() - proof, req, pid, tb := xcheckBuild(t, - "1111111111111111111111111111111111111111111111111111111111111111", - sid0, types.NetworkTestNet, types.NetworkMainNet) - require.EqualValues(t, types.NetworkMainNet, tb.GetNetworkID()) - - var uc types.UnicityCertificate - require.NoError(t, types.Cbor.Unmarshal(proof.UnicityCertificate, &uc)) - require.EqualValues(t, types.NetworkTestNet, uc.UnicitySeal.NetworkID) - - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, - }) - t.Logf("seal.NetworkID=%d vs trustBase.NetworkID=%d => Verify: %v", - uc.UnicitySeal.NetworkID, tb.GetNetworkID(), err) - require.NoError(t, err, "spec ensure(UC.C^r.alpha = T.alpha) would have rejected this") -} - -// ShardConfHash: the UC carries one, the verifier context does not, and the -// mismatch is simply not looked at. -func TestXCheck_ShardConfHashSkippable(t *testing.T) { - sid0, _ := types.ShardID{}.Split() - proof, req, pid, tb := xcheckBuild(t, - "1111111111111111111111111111111111111111111111111111111111111111", - sid0, types.NetworkMainNet, types.NetworkMainNet) - require.NoError(t, proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, ShardConfHash: nil, - })) - // Supplying any non-nil value does get compared -- so the check exists but - // is opt-in, and no production caller opts in. - err := proof.Verify(req, &VerifierContext{ - TrustBase: tb, PartitionID: pid, ExpectedShardID: sid0, - ShardConfHash: []byte{0xAA}, - }) - require.ErrorContains(t, err, "invalid shard configuration hash") - t.Logf("nil ShardConfHash: accepted; non-nil: %v", err) -} From 1eb74e2508ffd1fe1a89f618269b3dd212f479cd Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Tue, 25 Aug 2026 20:50:46 +0200 Subject: [PATCH 11/12] chore: remove leaked audit scaffolding; document the predicate CBOR tag Three more subagent scratch files reached the branch via 'git add -A': cmd/cfgdump/main.go (a config dumper), internal/signing/zzaudit_test.go and internal/signing/zzlens_predicate_test.go. The earlier cleanup missed them because it globbed zz_* and these are cfgdump and zz-without-underscore. Also fixes a real defect in the wire spec found in review: ownerPredicate was documented as a bare array, but Predicate.MarshalCBOR emits tag 39032 and Predicate.UnmarshalCBOR requires it, so a client following the document would reject or re-encode every proof's certification data. Verified on the wire: engine 1 / code 0x01 / params 0x02 encodes as d99878 83 01 4101 4102. Tag 39032 added to the registry. --- cmd/cfgdump/main.go | 31 ------ docs/inclusion-proof-wire.md | 10 +- internal/signing/zzaudit_test.go | 127 ---------------------- internal/signing/zzlens_predicate_test.go | 127 ---------------------- 4 files changed, 8 insertions(+), 287 deletions(-) delete mode 100644 cmd/cfgdump/main.go delete mode 100644 internal/signing/zzaudit_test.go delete mode 100644 internal/signing/zzlens_predicate_test.go diff --git a/cmd/cfgdump/main.go b/cmd/cfgdump/main.go deleted file mode 100644 index 7a1c18a1..00000000 --- a/cmd/cfgdump/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - - "github.com/unicitynetwork/aggregator-go/internal/config" -) - -func main() { - os.Setenv("BFT_ENABLED", "false") - c, err := config.Load() - if err != nil { - fmt.Println("ERR:", err) - os.Exit(1) - } - fmt.Printf("Chain.ForkID=%q Chain.ID=%q Chain.Version=%q\n", c.Chain.ForkID, c.Chain.ID, c.Chain.Version) - fmt.Printf("Server.EnableH2C=%v Server.HTTP2MaxConcurrentStreams=%d\n", c.Server.EnableH2C, c.Server.HTTP2MaxConcurrentStreams) - fmt.Printf("DB.FinalizationInsertChunkSize=%d Workers=%d\n", c.Database.FinalizationInsertChunkSize, c.Database.FinalizationInsertChunkWorkers) - fmt.Printf("Log.FilePath=%q MaxSizeMB=%d MaxBackups=%d MaxAgeDays=%d Compress=%v\n", c.Logging.FilePath, c.Logging.MaxSizeMB, c.Logging.MaxBackups, c.Logging.MaxAgeDays, c.Logging.CompressBackups) - fmt.Printf("Proc.BatchLimit=%d MaxCommitmentsPerRound=%d CollectPhase=%s MiniBatch=%d StreamBuf=%d Grace=%s SkipDup=%v TTL=%s\n", - c.Processing.BatchLimit, c.Processing.MaxCommitmentsPerRound, c.Processing.CollectPhaseDuration, c.Processing.CollectMiniBatchSize, - c.Processing.CommitmentStreamBufferSize, c.Processing.PrecollectorGracePeriod, c.Processing.SkipDuplicateCheck, c.Processing.DefaultRequestTTL) - fmt.Printf("Redis.PoolSize=%d MinIdle=%d MaxRetries=%d Dial=%s Read=%s Write=%s\n", c.Redis.PoolSize, c.Redis.MinIdleConns, c.Redis.MaxRetries, c.Redis.DialTimeout, c.Redis.ReadTimeout, c.Redis.WriteTimeout) - fmt.Printf("Storage.AckBatch=%d DeleteAfterAck=%v Cleanup=%s MaxStreamLen=%d MaxBatch=%d Flush=%s\n", c.Storage.RedisAckBatchSize, c.Storage.RedisDeleteAfterAck, c.Storage.RedisCleanupInterval, c.Storage.RedisMaxStreamLength, c.Storage.RedisMaxBatchSize, c.Storage.RedisFlushInterval) - fmt.Printf("SMT.PrecomputeProofs=%v ProofMetadataCacheEntries=%d MaterializeWorkers=%d\n", c.SMT.PrecomputeProofs, c.SMT.ProofMetadataCacheEntries, c.SMT.MaterializeWorkers) - b, _ := json.MarshalIndent(c.Sharding, "", " ") - fmt.Printf("Sharding=%s\n", b) - fmt.Printf("Signing.KeyFile=%q\n", c.Signing.KeyFile) -} diff --git a/docs/inclusion-proof-wire.md b/docs/inclusion-proof-wire.md index ddd56c04..bcdec5db 100644 --- a/docs/inclusion-proof-wire.md +++ b/docs/inclusion-proof-wire.md @@ -16,6 +16,7 @@ correct -- it does not present an implementation gap as a specification. |-----|-----------| | 39030 | `CertificationRequest` | | 39031 | `CertificationData` | +| 39032 | `Predicate` | | 39033 | `InclusionProofV2` | ## RPC response @@ -49,16 +50,21 @@ A tagged 6-element array. The element count never varies with the payload: | Index | Field | Type | |-------|-------|------| | 0 | `version` | uint, `2` | -| 1 | `ownerPredicate` | array | +| 1 | `ownerPredicate` | `#39032([engine: uint, code: bstr, params: bstr])` | | 2 | `sourceStateHash` | bstr(32) | | 3 | `transactionHash` | bstr(32) | | 4 | `expiresAt` | uint \| null | | 5 | `witness` | bstr(65) | +`ownerPredicate` is **tagged**, not a bare array: `Predicate.MarshalCBOR` emits +tag 39032 and `Predicate.UnmarshalCBOR` requires it. A predicate with engine 1, +code `0x01` and params `0x02` encodes as `d99878 83 01 4101 4102`. + `expiresAt` is the exclusive request deadline τ_Q. It holds its position and is written as CBOR `null` when the requester supplied no deadline, so the array length never depends on the payload. Absence is distinct from zero: zero is a -legal instant. +legal instant. Both forms are specified — the yellowpaper's request timeout is +optional, and `⊥` is written as CBOR null at a fixed position. ## Leaf value diff --git a/internal/signing/zzaudit_test.go b/internal/signing/zzaudit_test.go deleted file mode 100644 index 8dbcd66e..00000000 --- a/internal/signing/zzaudit_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package signing - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "testing" - - "github.com/btcsuite/btcd/btcec/v2" - "github.com/stretchr/testify/require" - - "github.com/unicitynetwork/aggregator-go/internal/config" - "github.com/unicitynetwork/aggregator-go/internal/models" - "github.com/unicitynetwork/aggregator-go/pkg/api" - bfttypes "github.com/unicitynetwork/bft-go-base/types" -) - -// AUDIT: byte-level verification of m, sid, lambda and predicate acceptance. -func TestAudit_Bytes(t *testing.T) { - priv, err := btcec.NewPrivateKey() - require.NoError(t, err) - pk := priv.PubKey().SerializeCompressed() - - sth := sha256.Sum256([]byte("sthash")) - txh := sha256.Sum256([]byte("txhash")) - - // ---- m = H(sthash, txhash) - m := api.SigDataHash(sth[:], txh[:]) - preM := append([]byte{0x82, 0x58, 0x20}, sth[:]...) - preM = append(preM, 0x58, 0x20) - preM = append(preM, txh[:]...) - wantM := sha256.Sum256(preM) - fmt.Printf("m preimage = %x\n", preM) - fmt.Printf("m = %x\n", m.RawHash) - fmt.Printf("m expected = %x\n", wantM) - require.Equal(t, wantM[:], m.RawHash) - - // ---- sid = H(pred, sthash) - pred := api.NewPayToPublicKeyPredicate(pk) - sid, err := api.CreateStateID(pred, sth[:]) - require.NoError(t, err) - // expected preimage: 82 d9 98 78 83 01 41 01 58 21 58 20 - preS := []byte{0x82, 0xd9, 0x98, 0x78, 0x83, 0x01, 0x41, 0x01, 0x58, 0x21} - preS = append(preS, pk...) - preS = append(preS, 0x58, 0x20) - preS = append(preS, sth[:]...) - wantS := sha256.Sum256(preS) - fmt.Printf("sid preimage = %x\n", preS) - fmt.Printf("sid = %x\n", sid) - fmt.Printf("sid expected = %x\n", wantS) - require.Equal(t, wantS[:], []byte(sid)) - - // ---- lambda(Q, tau) = H(txhash, tau) - lv := api.LeafValue(txh[:], 1755000000) - preL := append([]byte{0x82, 0x58, 0x20}, txh[:]...) - preL = append(preL, 0x1a, 0x68, 0x9b, 0x2c, 0xc0) // uint32 1755000000 - wantL := sha256.Sum256(preL) - fmt.Printf("leaf preimage = %x\n", preL) - fmt.Printf("leaf = %x\n", lv) - fmt.Printf("leaf expected = %x\n", wantL) - require.Equal(t, wantL[:], lv) -} - -func auditCommitment(t *testing.T, pred api.Predicate, priv *btcec.PrivateKey) *models.CertificationRequest { - t.Helper() - sth := sha256.Sum256([]byte("sthash")) - txh := sha256.Sum256([]byte("txhash")) - sid, err := api.CreateStateID(pred, sth[:]) - require.NoError(t, err) - sig, err := NewSigningService().SignDataHash(api.SigDataHash(sth[:], txh[:]), priv.Serialize()) - require.NoError(t, err) - return &models.CertificationRequest{ - StateID: sid, - CertificationData: models.CertificationData{ - OwnerPredicate: pred, - SourceStateHash: sth[:], - TransactionHash: txh[:], - Witness: api.HexBytes(sig), - }, - } -} - -// AUDIT: every non-0x01 predicate type code is rejected outright. -func TestAudit_PredicateTypeCodes(t *testing.T) { - v := NewCertificationRequestValidator(config.ShardingConfig{Mode: config.ShardingModeStandalone}, bfttypes.ShardID{}) - priv, err := btcec.NewPrivateKey() - require.NoError(t, err) - pk := priv.PubKey().SerializeCompressed() - - for code := 0x01; code <= 0x08; code++ { - pred := api.Predicate{Engine: 1, Code: []byte{byte(code)}, Params: pk} - res := v.Validate(auditCommitment(t, pred, priv)) - fmt.Printf("code 0x%02x -> status=%d string=%q err=%v\n", code, res.Status, res.Status.String(), res.Error) - } - // engine variations - for _, eng := range []uint{0, 2, 7} { - pred := api.Predicate{Engine: eng, Code: []byte{1}, Params: pk} - res := v.Validate(auditCommitment(t, pred, priv)) - fmt.Printf("engine %d -> status=%d string=%q err=%v\n", eng, res.Status, res.Status.String(), res.Error) - } - fmt.Printf("InvalidOwnerPredicate iota=%d String()=%q\n", - ValidationStatusInvalidOwnerPredicate, ValidationStatusInvalidOwnerPredicate.String()) - require.Equal(t, "UNKNOWN", ValidationStatusInvalidOwnerPredicate.String()) -} - -// AUDIT: does the validator take tau anywhere? -func TestAudit_NoTau(t *testing.T) { - // compile-time proof that Validate has exactly one parameter and no tau - var f func(*models.CertificationRequest) ValidationResult = (&CertificationRequestValidator{}).Validate - _ = f - fmt.Println("Validate signature: func(*models.CertificationRequest) ValidationResult -- no tau") -} - -// AUDIT: shard comparator is MSB-first prefix match on the sid bytes. -func TestAudit_ShardComparator(t *testing.T) { - // build shard id "0" and "1" (1-bit split) - id0, id1 := bfttypes.ShardID{}.Split() - for _, id := range []bfttypes.ShardID{id0, id1} { - cmp := id.Comparator() - key0 := make([]byte, 32) // 0x00.. -> top bit 0 - key1 := make([]byte, 32) - key1[0] = 0x80 // top bit 1 - fmt.Printf("shard %v (len=%d): key 0x00..=%v key 0x80..=%v\n", - id.String(), id.Length(), cmp(key0), cmp(key1)) - } - _ = hex.EncodeToString -} diff --git a/internal/signing/zzlens_predicate_test.go b/internal/signing/zzlens_predicate_test.go deleted file mode 100644 index de8b643d..00000000 --- a/internal/signing/zzlens_predicate_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package signing - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "testing" - - "github.com/btcsuite/btcd/btcec/v2" - "github.com/stretchr/testify/require" - "github.com/unicitynetwork/bft-go-base/types" - - "github.com/unicitynetwork/aggregator-go/internal/models" - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// Demonstrates: a well-formed request whose current-owner predicate is one of -// the yellowpaper's built-in codes other than 0x01 is rejected outright by the -// Unicity Service, with a status string clients see as "UNKNOWN". -func TestLens_NonSigBuiltinPredicatesRejected(t *testing.T) { - validator := newDefaultCertificationRequestValidator() - - priv, err := btcec.NewPrivateKey() - require.NoError(t, err) - pk := priv.PubKey().SerializeCompressed() - - sourceStateHash := CreateDataHash([]byte("state")) - txDataHash := CreateDataHash([]byte("tx")) - txHash := txDataHash.Imprint() - - // params encodings per appendix-token.tex:53-63 - tlockParams, err := types.Cbor.Marshal([]interface{}{pk, uint64(1000)}) - require.NoError(t, err) - pkh := sha256.Sum256(pk) - msigParams, err := types.Cbor.Marshal([]interface{}{pk}) - require.NoError(t, err) - tsigParams, err := types.Cbor.Marshal([]interface{}{uint64(1), []interface{}{pk}}) - require.NoError(t, err) - y := sha256.Sum256([]byte("preimage")) - htlcParams, err := types.Cbor.Marshal([]interface{}{pk, pk, y[:], uint64(2000)}) - require.NoError(t, err) - - cases := []struct { - name string - code byte - params []byte - }{ - {"0x02 burn", 0x02, pkh[:]}, - {"0x03 tlock", 0x03, tlockParams}, - {"0x04 p2pkh", 0x04, pkh[:]}, - {"0x05 p2sh", 0x05, pkh[:]}, - {"0x06 msig", 0x06, msigParams}, - {"0x07 tsig", 0x07, tsigParams}, - {"0x08 htlc", 0x08, htlcParams}, - } - - for _, c := range cases { - pred := api.Predicate{Engine: 1, Code: []byte{c.code}, Params: c.params} - stateID, err := api.CreateStateID(pred, sourceStateHash) - require.NoError(t, err) - - // A genuinely satisfying unlocking argument for the sig-shaped paths: - // signature over m = H(sthash, txhash). - sigDataHash := api.SigDataHash(sourceStateHash, txHash) - sig, err := NewSigningService().SignDataHash(sigDataHash, priv.Serialize()) - require.NoError(t, err) - - req := &models.CertificationRequest{ - StateID: stateID, - CertificationData: models.CertificationData{ - OwnerPredicate: pred, - SourceStateHash: sourceStateHash, - TransactionHash: txDataHash, - Witness: api.HexBytes(sig), - }, - } - res := validator.Validate(req) - fmt.Printf("%-12s -> status=%d string=%q err=%v\n", c.name, res.Status, res.Status.String(), res.Error) - require.Equal(t, ValidationStatusInvalidOwnerPredicate, res.Status) - require.Equal(t, "UNKNOWN", res.Status.String()) - } -} - -// Demonstrates the exact bytes hashed for sid and for m. -func TestLens_SidAndMPreimages(t *testing.T) { - pk, err := hex.DecodeString("02" + "11"+"22"+"33"+"44"+"55"+"66"+"77"+"88"+"99"+"aa"+"bb"+"cc"+"dd"+"ee"+"ff"+"00"+"11"+"22"+"33"+"44"+"55"+"66"+"77"+"88"+"99"+"aa"+"bb"+"cc"+"dd"+"ee"+"ff"+"00") - require.NoError(t, err) - pred := api.NewPayToPublicKeyPredicate(pk) - - sth := make([]byte, 32) - for i := range sth { - sth[i] = byte(i) - } - txh := make([]byte, 32) - for i := range txh { - txh[i] = byte(0x80 + i) - } - - type stateIDInput struct { - _ struct{} `cbor:",toarray"` - OwnerPredicate api.Predicate - SourceStateHash []byte - } - b, err := types.Cbor.Marshal(stateIDInput{OwnerPredicate: pred, SourceStateHash: sth}) - require.NoError(t, err) - fmt.Printf("sid preimage = %x\n", b) - sid, err := api.CreateStateID(pred, sth) - require.NoError(t, err) - fmt.Printf("sid = %x\n", []byte(sid)) - h := sha256.Sum256(b) - require.Equal(t, h[:], []byte(sid)) - - mPre := append([]byte{0x82, 0x58, 0x20}, sth...) - mPre = append(mPre, 0x58, 0x20) - mPre = append(mPre, txh...) - mh := sha256.Sum256(mPre) - fmt.Printf("m preimage = %x\nm = %x\n", mPre, mh[:]) - require.Equal(t, mh[:], api.SigDataHash(sth, txh).RawHash) - - // Leaf value lambda(Q,tau) = H(CBOR([txhash, tau])) - lv := api.LeafValue(txh, 1700000000) - lvPre := append([]byte{0x82, 0x58, 0x20}, txh...) - lvPre = append(lvPre, 0x1a, 0x65, 0x53, 0xf1, 0x00) - lvh := sha256.Sum256(lvPre) - fmt.Printf("leaf preimage= %x\n", lvPre) - require.Equal(t, lvh[:], lv) -} From c767749302dbf55779385cfe312c82f96d5c3675 Mon Sep 17 00:00:00 2001 From: Risto Laanoja Date: Wed, 26 Aug 2026 13:41:02 +0300 Subject: [PATCH 12/12] fix(api): align inclusion proof verification with SDKs --- docs/inclusion-proof-wire.md | 55 +++----- pkg/api/inclusion_proof_v2_verify_test.go | 152 +++++++++++++++++++--- pkg/api/types.go | 33 ++++- 3 files changed, 182 insertions(+), 58 deletions(-) diff --git a/docs/inclusion-proof-wire.md b/docs/inclusion-proof-wire.md index bcdec5db..858b2522 100644 --- a/docs/inclusion-proof-wire.md +++ b/docs/inclusion-proof-wire.md @@ -181,51 +181,26 @@ child. 1. Non-nil proof, request, verifier context and trust base. 2. `certificationData != null`, else non-inclusion (unimplemented). 3. Request `transactionHash` present, and equal to the proof's. -4. `expiresAt` equal on both sides, treating absence as a value of its own. +4. The proof and request have equal `expiresAt`, owner predicate, source state + hash and witness fields. 5. `UC.IR.h` extractable and exactly 32 bytes. -6. `referenceTime` present. -7. If `expiresAt` is present, `referenceTime < expiresAt`. **Exclusive**: a leaf +6. The request `stateId` is exactly the value derived from the certification + data's owner predicate and source state hash. +7. `referenceTime` present. +8. If `expiresAt` is present, `referenceTime < expiresAt`. **Exclusive**: a leaf created at exactly the deadline is expired. When `expiresAt` is absent this check cannot run — the service-assigned deadline is not carried in the proof, is not signed, and is not checkable by any later verifier. -8. `InclusionCert.Verify(key, LeafValue(txhash, referenceTime), UC.IR.h)`. -9. Unicity Certificate verification against the trust base. +9. `InclusionCert.Verify(key, LeafValue(txhash, referenceTime), UC.IR.h)`. +10. The UC's certified shard is a bit prefix of `stateId`. +11. The UC seal network equals the trust-base network. +12. Unicity Certificate verification against the expected partition, shard, + optional shard-configuration hash, and trust base. The nil-guard error strings are part of the public contract so reference verifiers in other languages can pin them. -## Known divergence from the yellowpaper: the shard binding is not checked - -**This is a soundness gap, not a caller responsibility.** An earlier revision of -this document told integrators to "derive `ExpectedShardID` from configuration". -That is not the specified check and does not close the hole. - -`platform.tex` `VerifyInclusionProof` takes the partition description `CD_β` as -an input and mandates, before any tree check: - -``` -ensure(UC.C^r.α = T.α) -σ ← UC.C^shard.σ -ensure(σ ∈ CD_β.SH) -ensure(f_{CD_β.SH}(sid) = σ) // Proof comes from the right shard -``` - -`f_SH` derives the expected shard **from the key itself**. `platform.tex` -explicitly forecloses delegating this to certificate verification: -`VerifyUnicityCert` "does not, by itself, prove that a particular state -identifier belongs to the shard named in `C^shard`; that binding is checked by -the proof verification functions below." - -`InclusionProofV2.Verify` implements the tree and certificate steps but not the -binding: it compares the UC's shard against a caller-supplied -`VerifierContext.ExpectedShardID` rather than computing `f_SH(sid)`. In a -multi-shard deployment a leaf whose key routes to shard A, committed in shard -B's SMT under shard B's validly signed UC, therefore verifies — reproduced in -testing. That is cross-shard double-spend exposure. The network id -`UC.C^r.α = T.α` is likewise unchecked, so a certificate sealed for one network -verifies against another network's trust base. - -Until `VerifierContext` carries the sharding scheme and `Verify` derives the -expected shard from `sid`, do not rely on this function alone for cross-shard -safety. `api.MatchesShardPrefix` implements `f_SH` and the admission path -applies it correctly. +The shard and network checks match the JavaScript, Java and Rust state-transition +SDK verifiers. `ExpectedShardID` remains an additional caller policy check for +backward compatibility; it does not replace deriving the state ID or checking +that the UC's certified shard contains it. diff --git a/pkg/api/inclusion_proof_v2_verify_test.go b/pkg/api/inclusion_proof_v2_verify_test.go index 77131dcd..0ce61025 100644 --- a/pkg/api/inclusion_proof_v2_verify_test.go +++ b/pkg/api/inclusion_proof_v2_verify_test.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "crypto" "errors" "testing" @@ -206,7 +207,7 @@ func TestInclusionProofV2Verify_RejectsInvalidUCInputRecordHash(t *testing.T) { // same root is placed into InputRecord.Hash and flows through the shard tree // and unicity tree verbatim. No aggregator plumbing is involved — this builds // the exact cryptographic objects that a real deployment would emit. -func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( +func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID, sealNetwork types.NetworkID) ( *InclusionProofV2, *CertificationRequest, types.PartitionID, @@ -214,16 +215,37 @@ func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( ) { t.Helper() - // A deterministic 32-byte state ID and tx hash. Content is irrelevant - // — Verify checks the cryptographic chain, not the values. - stateID := RequireNewImprintV2("1111111111111111111111111111111111111111111111111111111111111111") txHash := RequireNewImprintV2("2222222222222222222222222222222222222222222222222222222222222222") + certData := CertificationData{ + Version: CertificationDataVersion, + OwnerPredicate: Predicate{ + Engine: 1, + Code: []byte{1}, + Params: bytes.Repeat([]byte{0x02}, 33), + }, + SourceStateHash: bytes.Repeat([]byte{0x33}, StateTreeKeyLengthBytes), + TransactionHash: txHash, + Witness: bytes.Repeat([]byte{0x44}, 65), + } + var ( + stateID StateID + err error + ) + for candidate := 0; candidate < 256; candidate++ { + certData.SourceStateHash[len(certData.SourceStateHash)-1] = byte(candidate) + stateID, err = certData.CreateStateID() + require.NoError(t, err) + if stateID.DataBytes()[0]&0x80 == 0 { + break + } + } + // This fixture intentionally routes to shard 0. Passing ownerShard=shard 1 + // creates a fully signed wrong-shard proof for the regression test below. + require.Zero(t, stateID.DataBytes()[0]&0x80) req := &CertificationRequest{ - StateID: stateID, - CertificationData: CertificationData{ - TransactionHash: txHash, - }, + StateID: stateID, + CertificationData: certData, } // Single-leaf root: H(0x00 || key || value) under the v2 hash algorithm, @@ -247,10 +269,19 @@ func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( sid0, sid1 := types.ShardID{}.Split() const partitionID types.PartitionID = 0x0f0f0f0f + ir0Hash := test.RandomBytes(32) + ir1Hash := test.RandomBytes(32) + if ownerShard.Equal(sid0) { + ir0Hash = leafRoot + } else if ownerShard.Equal(sid1) { + ir1Hash = leafRoot + } else { + t.Fatalf("owner shard %s is outside the fixture's two-shard scheme", ownerShard) + } ir0 := &types.InputRecord{ Version: 1, PreviousHash: []byte{0, 0, 1}, - Hash: leafRoot, + Hash: ir0Hash, BlockHash: []byte{0, 0, 3}, SummaryValue: []byte{0, 0, 4}, Timestamp: types.NewTimestamp(), @@ -258,9 +289,6 @@ func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( Epoch: 0, SumOfEarnedFees: 0, } - // Shard 1 needs its own (non-degenerate) IR so the shard tree is - // well-formed; its hash is irrelevant to this test. - ir1Hash := test.RandomBytes(32) ir1 := &types.InputRecord{ Version: 1, PreviousHash: []byte{0, 0, 5}, @@ -312,6 +340,7 @@ func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( seal := &types.UnicitySeal{ Version: 1, + NetworkID: sealNetwork, RootChainRoundNumber: 1, Timestamp: types.NewTimestamp(), PreviousHash: test.RandomBytes(32), @@ -345,7 +374,7 @@ func buildSignedSingleLeafProof(t *testing.T, ownerShard types.ShardID) ( func TestInclusionProofV2Verify_HappyPath_FullySignedUC(t *testing.T) { sid0, _ := types.ShardID{}.Split() - proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) vctx := &VerifierContext{ TrustBase: tb, @@ -360,7 +389,7 @@ func TestInclusionProofV2Verify_HappyPath_FullySignedUC(t *testing.T) { func TestInclusionProofV2Verify_ShardMismatch_Rejected(t *testing.T) { sid0, sid1 := types.ShardID{}.Split() - proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) vctx := &VerifierContext{ TrustBase: tb, @@ -373,6 +402,97 @@ func TestInclusionProofV2Verify_ShardMismatch_Rejected(t *testing.T) { require.Contains(t, err.Error(), "invalid shard ID") } +// A validly signed shard-1 UC does not certify a state ID whose first bit +// routes it to shard 0, even when the caller also says it expects shard 1. +func TestInclusionProofV2Verify_StateIDMustBelongToCertifiedShard(t *testing.T) { + _, sid1 := types.ShardID{}.Split() + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid1, types.NetworkMainNet) + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid1, + }) + require.EqualError(t, err, "stateId does not belong to certified shard") +} + +// The seal is signed by a key accepted by the trust base, but its network is +// different. Signature validity must not substitute for network binding. +func TestInclusionProofV2Verify_SealNetworkMustMatchTrustBase(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkLocal) + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + }) + require.EqualError(t, err, "unicity seal network does not match trust base") +} + +func TestInclusionProofV2Verify_StateIDMustMatchCertificationData(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) + req.StateID = RequireNewImprintV2("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + }) + require.EqualError(t, err, "stateId does not match certification data") +} + +func TestInclusionProofV2Verify_CertificationDataMustMatchRequest(t *testing.T) { + sid0, _ := types.ShardID{}.Split() + tests := []struct { + name string + mutate func(*CertificationData) + errText string + }{ + { + name: "owner predicate", + mutate: func(cd *CertificationData) { + cd.OwnerPredicate.Params = append([]byte(nil), cd.OwnerPredicate.Params...) + cd.OwnerPredicate.Params[0] ^= 0xff + }, + errText: "proof certification data owner predicate does not match certification request owner predicate", + }, + { + name: "source state hash", + mutate: func(cd *CertificationData) { + cd.SourceStateHash = append([]byte(nil), cd.SourceStateHash...) + cd.SourceStateHash[0] ^= 0xff + }, + errText: "proof certification data source state hash does not match certification request source state hash", + }, + { + name: "witness", + mutate: func(cd *CertificationData) { + cd.Witness = append([]byte(nil), cd.Witness...) + cd.Witness[0] ^= 0xff + }, + errText: "proof certification data witness does not match certification request witness", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) + detached := *proof.CertificationData + proof.CertificationData = &detached + tt.mutate(proof.CertificationData) + + err := proof.Verify(req, &VerifierContext{ + TrustBase: tb, + PartitionID: partitionID, + ExpectedShardID: sid0, + }) + require.EqualError(t, err, tt.errText) + }) + } +} + // The request deadline is exclusive: a leaf created at exactly the deadline is // expired, one created a second earlier is not. The deadline does not enter the // leaf value, so the cryptographic chain is unaffected either way and this @@ -392,7 +512,7 @@ func TestInclusionProofV2Verify_ExpiryBoundaryIsExclusive(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) require.NotNil(t, proof.ReferenceTime) // Both copies must agree or Verify rejects on the equality check @@ -432,7 +552,7 @@ func TestInclusionProofV2Verify_ExpiryPresenceMustMatch(t *testing.T) { {"present on both but different", Uint64Ptr(1755003600), Uint64Ptr(1755003601)}, } { t.Run(tt.name, func(t *testing.T) { - proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0) + proof, req, partitionID, tb := buildSignedSingleLeafProof(t, sid0, types.NetworkMainNet) detached := *proof.CertificationData proof.CertificationData = &detached diff --git a/pkg/api/types.go b/pkg/api/types.go index 564cadb6..296a5363 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -385,8 +385,9 @@ type VerifierContext struct { } // Verify checks a v2 inclusion proof end-to-end against the outer -// CertificationRequest and VerifierContext: local SMT path, UnicityCertificate -// (shard tree → unicity tree → seal), and ShardTreeCertificate.Shard equality. +// CertificationRequest and VerifierContext: request and state-ID binding, local +// SMT path, certified-shard binding, network binding, and UnicityCertificate +// verification (shard tree → unicity tree → seal). // // The nil-guard error strings below are part of the public contract so // reference verifiers in other languages can pin them. @@ -418,6 +419,17 @@ func (p *InclusionProofV2) Verify(v2 *CertificationRequest, vctx *VerifierContex if !equalExpiresAt(p.CertificationData.ExpiresAt, v2.CertificationData.ExpiresAt) { return errors.New("proof certification data expiry does not match certification request expiry") } + if p.CertificationData.OwnerPredicate.Engine != v2.CertificationData.OwnerPredicate.Engine || + !bytes.Equal(p.CertificationData.OwnerPredicate.Code, v2.CertificationData.OwnerPredicate.Code) || + !bytes.Equal(p.CertificationData.OwnerPredicate.Params, v2.CertificationData.OwnerPredicate.Params) { + return errors.New("proof certification data owner predicate does not match certification request owner predicate") + } + if !bytes.Equal(p.CertificationData.SourceStateHash, v2.CertificationData.SourceStateHash) { + return errors.New("proof certification data source state hash does not match certification request source state hash") + } + if !bytes.Equal(p.CertificationData.Witness, v2.CertificationData.Witness) { + return errors.New("proof certification data witness does not match certification request witness") + } rootRaw, err := p.UCInputRecordHashRaw() if err != nil { @@ -432,6 +444,13 @@ func (p *InclusionProofV2) Verify(v2 *CertificationRequest, vctx *VerifierContex if err != nil { return fmt.Errorf("failed to derive SMT key from stateId: %w", err) } + expectedStateID, err := p.CertificationData.CreateStateID() + if err != nil { + return fmt.Errorf("failed to derive stateId from certification data: %w", err) + } + if !bytes.Equal(key, expectedStateID.DataBytes()) { + return errors.New("stateId does not match certification data") + } if p.ReferenceTime == nil { return errors.New("missing inclusion proof reference time") } @@ -449,6 +468,16 @@ func (p *InclusionProofV2) Verify(v2 *CertificationRequest, vctx *VerifierContex if err := types.Cbor.Unmarshal(p.UnicityCertificate, &uc); err != nil { return fmt.Errorf("failed to decode unicity certificate: %w", err) } + if uc.ShardTreeCertificate.Shard.Length() > uint(len(key)*8) || + !uc.ShardTreeCertificate.Shard.Comparator()(key) { + return errors.New("stateId does not belong to certified shard") + } + if uc.UnicitySeal == nil { + return errors.New("unicity certificate missing unicity seal") + } + if uc.UnicitySeal.NetworkID != vctx.TrustBase.GetNetworkID() { + return errors.New("unicity seal network does not match trust base") + } if err := uc.Verify(vctx.TrustBase, crypto.SHA256, vctx.PartitionID, vctx.ExpectedShardID, vctx.ShardConfHash); err != nil { return fmt.Errorf("unicity certificate verification failed: %w", err) }