From e9862d5a81ae3a6bde2a0d4d16de1ecd8f8fe66c Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Fri, 21 Aug 2026 00:23:21 -0600 Subject: [PATCH 1/2] feat(state)!: allow bounding the address UTXO scan by height and count `ReadRequest::UtxosByAddresses` becomes a struct variant carrying a `height_range` and an optional `max_entries`, and both are pushed down to the finalized address index scan. The index is keyed by `(AddressLocation, OutputLocation)`, so a start height is a seek and a limit is a `take`. Two parts are easy to get wrong, and the new tests cover them. The non-finalized chain can spend UTXOs that the finalized query returned, so a limited finalized scan over-fetches by the number of UTXOs the chain spends for those addresses. Without that, a full page of results can come back short, and a client paging by count reads a short page as the end of the address' UTXOs. Truncation happens after the finalized and non-finalized sets are merged, not while they are chained. Chaining two sorted maps is not globally sorted, and entries in the overlap window would otherwise be counted twice. No caller sets either bound yet: `get_address_utxos` passes the full height range and no limit, so this commit changes no behavior. --- .../zebra-state-breaking-20260818-000001.yaml | 4 + .../fuzz/fuzz_targets/rpc_handler_fuzz.rs | 2 +- zebra-rpc/src/methods.rs | 6 +- zebra-state/src/request.rs | 18 +- zebra-state/src/service.rs | 8 +- .../disk_format/transparent.rs | 62 +++---- .../src/service/finalized_state/zebra_db.rs | 9 +- .../zebra_db/block/tests/snapshot.rs | 6 +- .../finalized_state/zebra_db/transparent.rs | 71 ++++---- .../src/service/non_finalized_state/chain.rs | 13 ++ zebra-state/src/service/read/address/utxo.rs | 169 ++++++++++++++++-- 11 files changed, 281 insertions(+), 87 deletions(-) create mode 100644 .changes/unreleased/zebra-state-breaking-20260818-000001.yaml diff --git a/.changes/unreleased/zebra-state-breaking-20260818-000001.yaml b/.changes/unreleased/zebra-state-breaking-20260818-000001.yaml new file mode 100644 index 00000000000..4cc1e67d7bb --- /dev/null +++ b/.changes/unreleased/zebra-state-breaking-20260818-000001.yaml @@ -0,0 +1,4 @@ +project: zebra-state +kind: breaking +body: '`ReadRequest::UtxosByAddresses` is now a struct variant, taking a `height_range` and an optional `max_entries` alongside the addresses. Both bound the index scan, so the work the query does is set by what the caller asks for rather than by the size of the addresses'' UTXO sets ([#11239](https://github.com/ZcashFoundation/zebra/issues/11239)).' +time: 2026-08-18T00:00:01.000000000Z diff --git a/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs b/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs index ca87ac94cce..aa60c997034 100644 --- a/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs +++ b/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs @@ -411,7 +411,7 @@ impl Service for MockReadState { R::TransactionIdsByAddresses { .. } => { Ok(Resp::AddressesTransactionIds(Default::default())) } - R::UtxosByAddresses(_) => { + R::UtxosByAddresses { .. } => { // AddressUtxos requires an `AddressUtxos` struct; the field is // private to zebra-state so we cannot construct it without a // public constructor. Fall back to Err for this variant. diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index b48766e6cee..cc45625504c 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -2296,7 +2296,11 @@ where let valid_addresses = utxos_request.valid_addresses()?; // get utxos data for addresses - let request = zebra_state::ReadRequest::UtxosByAddresses(valid_addresses); + let request = zebra_state::ReadRequest::UtxosByAddresses { + addresses: valid_addresses, + height_range: Height::MIN..=Height::MAX, + max_entries: None, + }; let response = read_state .ready() .and_then(|service| service.call(request)) diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index b39ab62e90b..892826ca054 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -1423,10 +1423,22 @@ pub enum ReadRequest { #[cfg(feature = "indexer")] SpendingTransactionId(Spend), - /// Looks up utxos for the provided addresses. + /// Looks up utxos for the provided addresses, in the provided height range, + /// returning at most `max_entries` of them. /// /// Returns a type with found utxos and transaction information. - UtxosByAddresses(HashSet), + UtxosByAddresses { + /// The addresses to look up utxos for. + addresses: HashSet, + + /// The blocks to be queried for utxos. + height_range: RangeInclusive, + + /// The maximum number of utxos to return, or `None` for no limit. + /// + /// The limit bounds the index scan, not just the response. + max_entries: Option, + }, /// Contextually validates anchors and nullifiers of a transaction on the best chain /// @@ -1525,7 +1537,7 @@ impl ReadRequest { ReadRequest::IronwoodSubtrees { .. } => "ironwood_subtrees", ReadRequest::AddressBalance { .. } => "address_balance", ReadRequest::TransactionIdsByAddresses { .. } => "transaction_ids_by_addresses", - ReadRequest::UtxosByAddresses(_) => "utxos_by_addresses", + ReadRequest::UtxosByAddresses { .. } => "utxos_by_addresses", ReadRequest::CheckBestChainTipNullifiersAndAnchors(_) => { "best_chain_tip_nullifiers_anchors" } diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index ad62d32e127..2c12f54b554 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1691,11 +1691,17 @@ impl Service for ReadStateService { .map(ReadResponse::AddressesTransactionIds), // For the get_address_utxos RPC. - ReadRequest::UtxosByAddresses(addresses) => read::address_utxos( + ReadRequest::UtxosByAddresses { + addresses, + height_range, + max_entries, + } => read::address_utxos( &state.network, state.latest_best_chain(), &state.db, addresses, + height_range, + max_entries, ) .map(ReadResponse::AddressUtxos), diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index bf695381b70..9ad9220b21c 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -426,41 +426,41 @@ impl AddressUnspentOutput { } } - /// Create an [`AddressUnspentOutput`] which starts iteration for the - /// supplied address. Used to look up the first output with - /// [`ReadDisk::zs_next_key_value_from`][1]. + /// Create a range of [`AddressUnspentOutput`]s which covers the unspent outputs of the + /// supplied address in `query`. Starts at the first UTXO, or at the `query` start height, + /// whichever is greater. Ends at the maximum possible output index for the end height. /// - /// The unspent output location is before all unspent output locations in - /// the index. It is always invalid, due to the genesis consensus rules. But - /// this is not an issue since [`ReadDisk::zs_next_key_value_from`][1] will - /// fetch the next existing (valid) value. + /// Used to look up unspent outputs with [`DiskDb::zs_forward_range_iter`][1]. /// - /// [1]: super::super::disk_db::ReadDisk::zs_next_key_value_from - pub fn address_iterator_start(address_location: AddressLocation) -> AddressUnspentOutput { - // Iterating from the lowest possible output location gets us the first output. - let zero_output_location = OutputLocation::from_usize(Height(0), 0, 0); - - AddressUnspentOutput { - address_location, - unspent_output_location: zero_output_location, - } - } - - /// Update the unspent output location to the next possible output for the - /// supplied address. Used to look up the next output with - /// [`ReadDisk::zs_next_key_value_from`][1]. + /// The output locations in the: + /// - start bound might be invalid, if it is based on the `query` start height. + /// - end bound will always be invalid. /// - /// The updated unspent output location may be invalid, which is not an - /// issue since [`ReadDisk::zs_next_key_value_from`][1] will fetch the next - /// existing (valid) value. + /// But this is not an issue, since [`DiskDb::zs_forward_range_iter`][1] will fetch all existing + /// (valid) values in the range. /// - /// [1]: super::super::disk_db::ReadDisk::zs_next_key_value_from - pub fn address_iterator_next(&mut self) { - // Iterating from the next possible output location gets us the next output, - // even if it is in a later block or transaction. - // - // Consensus: the block size limit is 2MB, which is much lower than the index range. - self.unspent_output_location.output_index += 1; + /// [1]: super::super::disk_db::DiskDb + pub fn address_iterator_range( + address_location: AddressLocation, + query: std::ops::RangeInclusive, + ) -> std::ops::RangeInclusive { + // The address location is the output location of the first UTXO sent to the address, + // so no unspent output can be before it. + let first_utxo_location = address_location; + + // Iterating from the start height to the end height filters out outputs that aren't needed. + let query_start_location = OutputLocation::from_output_index( + TransactionLocation::from_index(*query.start(), 0), + 0, + ); + let query_end_location = OutputLocation::from_output_index( + TransactionLocation::from_index(*query.end(), u16::MAX), + u32::MAX, + ); + + let addr_out = |out_loc| AddressUnspentOutput::new(address_location, out_loc); + + addr_out(max(first_utxo_location, query_start_location))..=addr_out(query_end_location) } /// The location of the first [`transparent::Output`] sent to the address of this output. diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index 1612167629b..dbc9d7a739e 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -178,8 +178,13 @@ impl ZebraDb { db: disk_db, }; - let zero_location_utxos = - db.address_utxo_locations(AddressLocation::from_usize(Height(0), 0, 0)); + // One entry is enough to detect the corruption, and the height range has to start at + // zero, because the corrupt entries are exactly the ones at the zero address location. + let zero_location_utxos = db.address_utxo_locations( + AddressLocation::from_usize(Height(0), 0, 0), + Height(0)..=Height::MAX, + Some(1), + ); if !zero_location_utxos.is_empty() { warn!( "You have been impacted by the Zebra 2.4.0 address indexer corruption bug. \ diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index 566edd381ba..21ff94974a8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -512,14 +512,16 @@ fn snapshot_transparent_address_data(state: &FinalizedState, height: u32) { let stored_address_location = stored_address_balance_location.address_location(); let mut stored_utxo_locations = Vec::new(); - for address_utxo_loc in state.address_utxo_locations(stored_address_location) { + for address_utxo_loc in + state.address_utxo_locations(stored_address_location, Height(0)..=Height::MAX, None) + { assert_eq!(address_utxo_loc.address_location(), stored_address_location); stored_utxo_locations.push(address_utxo_loc.unspent_output_location()); } let mut stored_utxos = Vec::new(); - for (utxo_loc, utxo) in state.address_utxos(&address) { + for (utxo_loc, utxo) in state.address_utxos(&address, Height(0)..=Height::MAX, None) { assert!(stored_utxo_locations.contains(&utxo_loc)); stored_utxos.push(utxo); diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index d1eb7017126..a62b6d87c88 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -185,18 +185,29 @@ impl ZebraDb { Some(utxo) } - /// Returns the unspent transparent outputs for a [`transparent::Address`], - /// if they are in the finalized state. + /// Returns the unspent transparent outputs for a [`transparent::Address`] in the finalized + /// chain `query_height_range`, at most `limit` of them. + /// + /// If the address has no finalized UTXOs, or the `query_height_range` is totally outside + /// the finalized block range, returns an empty list. pub fn address_utxos( &self, address: &transparent::Address, + query_height_range: RangeInclusive, + limit: Option, ) -> BTreeMap { let address_location = match self.address_location(address) { Some(address_location) => address_location, None => return BTreeMap::new(), }; - let output_locations = self.address_utxo_locations(address_location); + // Skip this address if its first UTXO is after the end height. + if address_location.height() > *query_height_range.end() { + return BTreeMap::new(); + } + + let output_locations = + self.address_utxo_locations(address_location, query_height_range, limit); // Ignore any outputs spent by blocks committed during this query output_locations @@ -212,46 +223,29 @@ impl ZebraDb { .collect() } - /// Returns the unspent transparent output locations for a [`transparent::Address`], - /// if they are in the finalized state. + /// Returns the unspent transparent output locations for a [`transparent::Address`] in the + /// finalized chain `query_height_range`, at most `limit` of them. pub fn address_utxo_locations( &self, address_location: AddressLocation, + query_height_range: RangeInclusive, + limit: Option, ) -> BTreeSet { let utxo_loc_by_transparent_addr_loc = self .db .cf_handle("utxo_loc_by_transparent_addr_loc") .unwrap(); - // Manually fetch the entire addresses' UTXO locations - let mut addr_unspent_outputs = BTreeSet::new(); - - // An invalid key representing the minimum possible output - let mut unspent_output = AddressUnspentOutput::address_iterator_start(address_location); - - loop { - // Seek to a valid entry for this address, or the first entry for the next address - unspent_output = match self - .db - .zs_next_key_value_from(&utxo_loc_by_transparent_addr_loc, &unspent_output) - { - Some((unspent_output, ())) => unspent_output, - // We're finished with the final address in the column family - None => break, - }; - - // We found the next address, so we're finished with this address - if unspent_output.address_location() != address_location { - break; - } - - addr_unspent_outputs.insert(unspent_output); - - // A potentially invalid key representing the next possible output - unspent_output.address_iterator_next(); - } + // A potentially invalid key representing the first UTXO sent to the address, + // or the query start height. + let unspent_output_range = + AddressUnspentOutput::address_iterator_range(address_location, query_height_range); - addr_unspent_outputs + self.db + .zs_forward_range_iter(&utxo_loc_by_transparent_addr_loc, unspent_output_range) + .map(|(unspent_output, ())| unspent_output) + .take(limit.unwrap_or(usize::MAX)) + .collect() } /// Returns the transaction hash for an [`TransactionLocation`]. @@ -354,10 +348,15 @@ impl ZebraDb { ) } - /// Returns the UTXOs for `addresses` in the finalized chain. + /// Returns the UTXOs for `addresses` in the finalized chain `query_height_range`, + /// at most `limit` of them per address. /// /// If none of the addresses has finalized UTXOs, returns an empty list. /// + /// The per-address `limit` is enough for the caller to apply a global limit of the same + /// size: the first `limit` UTXOs across all addresses can only contain UTXOs that are + /// among the first `limit` for their own address. + /// /// # Correctness /// /// Callers should apply the non-finalized UTXO changes for `addresses` to the returned UTXOs. @@ -370,10 +369,12 @@ impl ZebraDb { pub fn partial_finalized_address_utxos( &self, addresses: &HashSet, + query_height_range: RangeInclusive, + limit: Option, ) -> BTreeMap { addresses .iter() - .flat_map(|address| self.address_utxos(address)) + .flat_map(|address| self.address_utxos(address, query_height_range.clone(), limit)) .collect() } diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index f327a57f3a3..928896fe856 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -1649,6 +1649,19 @@ impl Chain { (created_utxos, spent_utxos) } + /// Returns the number of UTXOs that `addresses` spend in this partial non-finalized chain. + /// + /// A limited finalized UTXO query has to over-fetch by this many entries, because any of + /// the UTXOs it returns can turn out to be spent here. + pub fn partial_transparent_spent_utxo_count( + &self, + addresses: &HashSet, + ) -> usize { + self.partial_transparent_indexes(addresses) + .map(|transfers| transfers.spent_utxos().len()) + .sum() + } + /// Returns the [`transaction::Hash`]es used by `addresses` to receive or spend funds, /// in the non-finalized chain, filtered using the `query_height_range`. /// diff --git a/zebra-state/src/service/read/address/utxo.rs b/zebra-state/src/service/read/address/utxo.rs index a9c4b38b553..8957a97456c 100644 --- a/zebra-state/src/service/read/address/utxo.rs +++ b/zebra-state/src/service/read/address/utxo.rs @@ -103,16 +103,22 @@ impl AddressUtxos { } } -/// Returns the unspent transparent outputs (UTXOs) for the supplied [`transparent::Address`]es, -/// in chain order; and the transaction IDs for the transactions containing those UTXOs. +/// Returns the unspent transparent outputs (UTXOs) for the supplied [`transparent::Address`]es +/// in `query_height_range`, in chain order, at most `max_entries` of them; and the transaction +/// IDs for the transactions containing those UTXOs. /// /// If the addresses do not exist in the non-finalized `chain` or finalized `db`, /// returns an empty list. +/// +/// Both limits are applied to the index scan, so the work this query does is bounded by what +/// the caller asks for, rather than by the size of the addresses' UTXO sets. pub fn address_utxos( network: &Network, chain: Option, db: &ZebraDb, addresses: HashSet, + query_height_range: RangeInclusive, + max_entries: Option, ) -> Result where C: AsRef, @@ -120,6 +126,22 @@ where let mut utxo_error = None; let address_count = addresses.len(); + // The non-finalized chain can spend UTXOs that the finalized query returns, so a limited + // finalized query has to over-fetch by that many entries, or a full page of results can + // arrive short and look like the end of the address' UTXOs. + let finalized_limit = max_entries.map(|max_entries| { + let chain_spent_count = chain + .as_ref() + .map(|chain| { + chain + .as_ref() + .partial_transparent_spent_utxo_count(&addresses) + }) + .unwrap_or(0); + + finalized_scan_limit(max_entries, chain_spent_count) + }); + // Retry the finalized UTXO query if it was interrupted by a finalizing block, // and the non-finalized chain doesn't overlap the changed heights. // @@ -127,7 +149,8 @@ where for attempt in 0..=FINALIZED_STATE_QUERY_RETRIES { debug!(?attempt, ?address_count, "starting address UTXO query"); - let (finalized_utxos, finalized_tip_range) = finalized_address_utxos(db, &addresses); + let (finalized_utxos, finalized_tip_range) = + finalized_address_utxos(db, &addresses, query_height_range.clone(), finalized_limit); debug!( finalized_utxo_count = ?finalized_utxos.len(), @@ -152,8 +175,14 @@ where "chain address UTXO response", ); - let utxos = - apply_utxo_changes(finalized_utxos, created_chain_utxos, spent_chain_utxos); + let utxos = apply_utxo_changes( + finalized_utxos, + created_chain_utxos, + spent_chain_utxos, + &query_height_range, + max_entries, + ); + let tx_ids = lookup_tx_ids_for_utxos(chain.as_ref(), db, &addresses, &utxos); debug!( @@ -207,6 +236,8 @@ where fn finalized_address_utxos( db: &ZebraDb, addresses: &HashSet, + query_height_range: RangeInclusive, + limit: Option, ) -> ( BTreeMap, Option>, @@ -218,7 +249,7 @@ fn finalized_address_utxos( // Check if the finalized state changed while we were querying it let start_finalized_tip = db.finalized_tip_height(); - let finalized_utxos = db.partial_finalized_address_utxos(addresses); + let finalized_utxos = db.partial_finalized_address_utxos(addresses, query_height_range, limit); let end_finalized_tip = db.finalized_tip_height(); @@ -400,20 +431,50 @@ where Ok((created, spent, Some(non_finalized_tip))) } -/// Combines the supplied finalized and non-finalized UTXOs, -/// removes the spent UTXOs, and returns the result. +/// The number of UTXOs a limited finalized query has to ask for, when the caller wants +/// `max_entries` of them and the non-finalized chain spends `chain_spent_count` of the +/// addresses' UTXOs. +/// +/// Any of the UTXOs the finalized query returns can turn out to be spent by the chain, so the +/// query has to over-fetch by as many entries as the chain spends. Otherwise a full page comes +/// back short, and a caller paging by count reads that as the end of the addresses' UTXOs. +/// +/// The result is an over-approximation: it can make the query read entries that are then +/// discarded, but never too few to fill the page. +fn finalized_scan_limit(max_entries: usize, chain_spent_count: usize) -> usize { + max_entries.saturating_add(chain_spent_count) +} + +/// Combines the supplied finalized and non-finalized UTXOs, removes the spent UTXOs and the +/// UTXOs outside `query_height_range`, truncates the result to `max_entries`, and returns it. fn apply_utxo_changes( finalized_utxos: BTreeMap, created_chain_utxos: BTreeMap, spent_chain_utxos: BTreeSet, + query_height_range: &RangeInclusive, + max_entries: Option, ) -> BTreeMap { // Correctness: combine the created UTXOs, then remove spent UTXOs, // to compensate for overlapping finalized and non-finalized blocks. - finalized_utxos + // + // The finalized UTXOs are already limited to the query height range, but the + // non-finalized ones are not. + let utxos: BTreeMap<_, _> = finalized_utxos .into_iter() - .chain(created_chain_utxos) + .chain( + created_chain_utxos + .into_iter() + .filter(|(utxo_location, _)| query_height_range.contains(&utxo_location.height())), + ) .filter(|(utxo_location, _output)| !spent_chain_utxos.contains(utxo_location)) - .collect() + .collect(); + + // Truncate after collecting, so the entries that survive are the first in chain order, + // rather than the first in the order the two sources happened to be chained in. + match max_entries { + Some(max_entries) => utxos.into_iter().take(max_entries).collect(), + None => utxos, + } } /// Returns the [`transaction::Hash`]es containing the supplied UTXOs, @@ -460,3 +521,89 @@ where }) .collect() } + +#[cfg(test)] +mod tests { + use zebra_chain::{amount::Amount, transparent::Script}; + + use super::*; + + /// Returns a UTXO at `height`, and its location. + fn utxo(height: u32) -> (OutputLocation, transparent::Output) { + ( + OutputLocation::from_usize(Height(height), 0, 0), + transparent::Output::new(Amount::zero(), Script::new(&[])), + ) + } + + /// Returns the heights of `utxos`, in chain order. + fn heights(utxos: &BTreeMap) -> Vec { + utxos.keys().map(|location| location.height().0).collect() + } + + /// The non-finalized chain creates UTXOs across the whole chain, not just the queried + /// range, so the ones outside it have to be dropped even though the finalized query + /// never returned them. + #[test] + fn created_chain_utxos_are_limited_to_the_query_height_range() { + let finalized = BTreeMap::from([utxo(30), utxo(40)]); + let created = BTreeMap::from([utxo(10), utxo(50)]); + + let utxos = apply_utxo_changes( + finalized, + created, + BTreeSet::new(), + &(Height(20)..=Height(45)), + None, + ); + + assert_eq!(heights(&utxos), vec![30, 40]); + } + + /// A truncated result keeps the first UTXOs in chain order, not the first in the order + /// the finalized and non-finalized sources are combined in. + #[test] + fn a_limited_result_keeps_the_lowest_utxos() { + let finalized = BTreeMap::from([utxo(30), utxo(40)]); + let created = BTreeMap::from([utxo(10), utxo(20)]); + + let utxos = apply_utxo_changes( + finalized, + created, + BTreeSet::new(), + &ADDRESS_HEIGHTS_FULL_RANGE, + Some(3), + ); + + assert_eq!(heights(&utxos), vec![10, 20, 30]); + } + + /// The over-fetch is what keeps a page full when the non-finalized chain spends some of + /// the UTXOs the finalized query returned. + #[test] + fn over_fetching_keeps_a_limited_page_full() { + let max_entries = 2; + let all_finalized = [utxo(10), utxo(20), utxo(30), utxo(40), utxo(50)]; + + // The chain spends the first UTXO the finalized query would return. + let spent = BTreeSet::from([all_finalized[0].0]); + + // The finalized query returns the first `limit` UTXOs, whatever the limit is. + let query = |limit: usize| { + apply_utxo_changes( + all_finalized.iter().take(limit).cloned().collect(), + BTreeMap::new(), + spent.clone(), + &ADDRESS_HEIGHTS_FULL_RANGE, + Some(max_entries), + ) + }; + + // Asking for exactly `max_entries` returns a short page, because one of them is spent. + assert_eq!(heights(&query(max_entries)), vec![20]); + + // Over-fetching by the number of spends fills the page instead. + let limit = finalized_scan_limit(max_entries, spent.len()); + assert_eq!(heights(&query(limit)), vec![20, 30]); + } +} From b7721750c69d0bc16abe087ac0199d7e6f4dcb76 Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Fri, 21 Aug 2026 00:25:12 -0600 Subject: [PATCH 2/2] fix(rpc)!: honor startHeight and maxEntries in getaddressutxos The `GetAddressUtxos` gRPC method accepted both arguments and applied them, but only to the finished reply: the node read every UTXO held by the named addresses first, then discarded whatever was not asked for. The `getaddressutxos` JSON-RPC underneath had no such arguments at all, so no caller could ask for less work. Pass both bounds into the state query, and drop the reply-side filter in the gRPC method. The cost is now set by what the caller asked for rather than by the size of the addresses' UTXO sets. That is the unremediated part of GHSA-x4m7-3gpp-xc36, and item 1 of #11239. A start height above the chain tip is clamped rather than rejected, because a start height above the chain already selects nothing, which is what the request means. Because the JSON-RPC previously ignored both fields, a client that already sends them now receives fewer entries than before for the same request. --- .../zebra-rpc-breaking-20260818-000002.yaml | 4 + .../fuzz_targets/jsonrpsee_envelope_fuzz.rs | 2 +- .../fuzz/fuzz_targets/rpc_handler_fuzz.rs | 13 ++- zebra-rpc/src/lightwalletd/methods.rs | 21 ++--- zebra-rpc/src/methods.rs | 88 ++++++++++++++---- zebra-rpc/src/methods/tests/vectors.rs | 89 +++++++++++++++++++ 6 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 .changes/unreleased/zebra-rpc-breaking-20260818-000002.yaml diff --git a/.changes/unreleased/zebra-rpc-breaking-20260818-000002.yaml b/.changes/unreleased/zebra-rpc-breaking-20260818-000002.yaml new file mode 100644 index 00000000000..f93865fc72f --- /dev/null +++ b/.changes/unreleased/zebra-rpc-breaking-20260818-000002.yaml @@ -0,0 +1,4 @@ +project: zebra-rpc +kind: breaking +body: '`getaddressutxos` now accepts optional `startHeight` and `maxEntries` fields and bounds the state query by them. The JSON-RPC had no such fields before and silently dropped them, so a client that already sends them now receives fewer entries than before for the same request. The `GetAddressUtxos` gRPC method already accepted both, but applied them to the finished reply after the node had read every UTXO held by the named addresses; it now passes them through, so a request naming one address with a large UTXO set no longer costs the same whether it asks for one entry or all of them ([#11239](https://github.com/ZcashFoundation/zebra/issues/11239)).' +time: 2026-08-18T00:00:02.000000000Z diff --git a/zebra-fuzz/fuzz/fuzz_targets/jsonrpsee_envelope_fuzz.rs b/zebra-fuzz/fuzz/fuzz_targets/jsonrpsee_envelope_fuzz.rs index 8dbcd66932d..17de61b75e2 100644 --- a/zebra-fuzz/fuzz/fuzz_targets/jsonrpsee_envelope_fuzz.rs +++ b/zebra-fuzz/fuzz/fuzz_targets/jsonrpsee_envelope_fuzz.rs @@ -532,7 +532,7 @@ fn build_envelope(data: &[u8]) -> Option { /// z_listunifiedreceivers(address: String) /// getaddressbalance(req: GetAddressBalanceRequest) // {"addresses":[...]} /// getaddresstxids(req: GetAddressTxIdsRequest) // {"addresses":[...], "start":N, "end":N} -/// getaddressutxos(req: GetAddressUtxosRequest) // {"addresses":[...]} OR {"chainInfo": false, "addresses":[...]} +/// getaddressutxos(req: GetAddressUtxosRequest) // {"addresses":[...]} OR {"chainInfo": false, "addresses":[...], "startHeight": 0, "maxEntries": 0} /// z_gettreestate(hash_or_height: String) /// z_getsubtreesbyindex(pool: String, start_index: u16, limit: Option) /// invalidateblock(block_hash: String) diff --git a/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs b/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs index aa60c997034..6e05156272f 100644 --- a/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs +++ b/zebra-fuzz/fuzz/fuzz_targets/rpc_handler_fuzz.rs @@ -181,8 +181,10 @@ enum RpcCall { /// raw fields constructor. GetAddressTxIds(Vec, Option, Option), /// `getaddressutxos(GetAddressUtxosRequest)` — methods.rs:2093. Two- - /// form DTO + `chain_info` bool. Same family as the two above. - GetAddressUtxos(Vec, bool), + /// form DTO + `chain_info` bool, and the `start_height` / `max_entries` + /// limits, which are pushed into the state index scan. Same family as + /// the two above. + GetAddressUtxos(Vec, bool, u64, u32), // ── Tier-B · treestate (hash_or_height + pool name) ──────────────── /// `z_gettreestate(hash_or_height)` — methods.rs:1874. Same @@ -762,8 +764,11 @@ async fn dispatch(rpc: &FuzzRpcImpl, call: RpcCall) { .catch_unwind() .await; } - RpcCall::GetAddressUtxos(addrs, chain_info) => { - let req = GetAddressUtxosRequest::new(addrs, chain_info); + RpcCall::GetAddressUtxos(addrs, chain_info, start_height, max_entries) => { + // `start_height` is passed through unclamped so the fuzzer can reach the + // heights above `Height::MAX` that the RPC has to clamp rather than reject. + let req = + GetAddressUtxosRequest::new(addrs, chain_info).with_limits(start_height, max_entries); let _ = AssertUnwindSafe(rpc.get_address_utxos(req)) .catch_unwind() .await; diff --git a/zebra-rpc/src/lightwalletd/methods.rs b/zebra-rpc/src/lightwalletd/methods.rs index ff31756f72d..3e009afbfbe 100644 --- a/zebra-rpc/src/lightwalletd/methods.rs +++ b/zebra-rpc/src/lightwalletd/methods.rs @@ -955,8 +955,11 @@ async fn address_balance( }) } -/// Fetches the UTXOs for a list of transparent addresses, filtered by the given -/// start height and maximum number of entries. +/// Fetches the UTXOs for a list of transparent addresses, at or above the given start height, +/// at most `max_entries` of them. +/// +/// Both limits are passed to the state query, so they bound the work the node does rather than +/// just the size of the reply. async fn address_utxos( rpc: &Rpc, args: GetAddressUtxosArg, @@ -964,7 +967,10 @@ async fn address_utxos( check_address_count(&args.addresses)?; let response = rpc - .get_address_utxos(GetAddressUtxosRequest::new(args.addresses, false)) + .get_address_utxos( + GetAddressUtxosRequest::new(args.addresses, false) + .with_limits(args.start_height, args.max_entries), + ) .await .map_err(rpc_arg_error_to_status)?; @@ -972,17 +978,8 @@ async fn address_utxos( unreachable!("chain info is never requested"); }; - let max_entries = if args.max_entries == 0 { - usize::MAX - } else { - // Cast is safe: `usize` is at least 32 bits on all supported platforms. - args.max_entries as usize - }; - Ok(utxos .iter() - .filter(|utxo| u64::from(utxo.height().0) >= args.start_height) - .take(max_entries) .map(|utxo| GetAddressUtxosReply { address: utxo.address().to_string(), txid: utxo.txid().0.to_vec(), diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index cc45625504c..ff413fc0a3a 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -473,9 +473,16 @@ pub trait Rpc { /// - An object with the following named fields: /// - `addresses`: (array, required, example=[\"tmYXBYJj1K7vhejSec5osXK2QsGa5MTisUQ\"]) The addresses to get outputs from. /// - `chaininfo`: (boolean, optional, default=false) Include chain info with results + /// - `startHeight`: (numeric, optional, default=0) Only return outputs at or above this height + /// - `maxEntries`: (numeric, optional, default=0) Return at most this many outputs, zero for no limit /// /// # Notes /// + /// `startHeight` and `maxEntries` bound the index scan, not just the response, so a client + /// that pages through a large address does not make the node read the whole UTXO set each + /// time. Callers must set both or neither: `maxEntries` alone would truncate the outputs + /// before the height filter ran. + /// /// lightwalletd always uses the multi-address request, without chaininfo: /// #[method(name = "getaddressutxos")] @@ -2295,11 +2302,22 @@ where let valid_addresses = utxos_request.valid_addresses()?; + // Clamp rather than error: a start height above the chain selects nothing, which is + // already what the request means. + let start_height = Height( + u32::try_from(utxos_request.start_height) + .unwrap_or(u32::MAX) + .min(Height::MAX.0), + ); + let max_entries = (utxos_request.max_entries > 0) + // Cast is safe: `usize` is at least 32 bits on all supported platforms. + .then_some(utxos_request.max_entries as usize); + // get utxos data for addresses let request = zebra_state::ReadRequest::UtxosByAddresses { addresses: valid_addresses, - height_range: Height::MIN..=Height::MAX, - max_entries: None, + height_range: start_height..=Height::MAX, + max_entries, }; let response = read_state .ready() @@ -3868,32 +3886,62 @@ pub use self::GetAddressBalanceResponse as AddressBalance; /// Parameters of [`RpcServer::get_address_utxos`] RPC method. #[derive( - Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize, Getters, new, JsonSchema, + Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize, Getters, JsonSchema, )] #[serde(from = "DGetAddressUtxosRequest")] pub struct GetAddressUtxosRequest { - /// A list of addresses to get transactions from. + /// A list of addresses to get unspent outputs from. addresses: Vec, - /// The height to start looking for transactions. + /// Whether to return chain info along with the unspent outputs. #[serde(default)] #[serde(rename = "chainInfo")] chain_info: bool, + /// The height to start looking for unspent outputs, or zero for the whole chain. + #[serde(default)] + #[serde(rename = "startHeight")] + start_height: u64, + /// The maximum number of unspent outputs to return, or zero for no limit. + #[serde(default)] + #[serde(rename = "maxEntries")] + max_entries: u32, +} + +impl GetAddressUtxosRequest { + /// Creates a new request for every unspent output held by `addresses`. + pub fn new(addresses: Vec, chain_info: bool) -> GetAddressUtxosRequest { + GetAddressUtxosRequest { + addresses, + chain_info, + start_height: 0, + max_entries: 0, + } + } + + /// Limits this request to the unspent outputs at or above `start_height`, and to at most + /// `max_entries` of them. Zero means "from the genesis block" and "no limit". + /// + /// Both limits are taken together, because applying `max_entries` on its own would + /// truncate the outputs before the height filter ran. + pub fn with_limits(self, start_height: u64, max_entries: u32) -> GetAddressUtxosRequest { + GetAddressUtxosRequest { + start_height, + max_entries, + ..self + } + } } impl From for GetAddressUtxosRequest { fn from(request: DGetAddressUtxosRequest) -> Self { match request { - DGetAddressUtxosRequest::Single(addr) => GetAddressUtxosRequest { - addresses: vec![addr], - chain_info: false, - }, + DGetAddressUtxosRequest::Single(addr) => GetAddressUtxosRequest::new(vec![addr], false), DGetAddressUtxosRequest::Object { addresses, chain_info, - } => GetAddressUtxosRequest { - addresses, - chain_info, - }, + start_height, + max_entries, + } => GetAddressUtxosRequest::new(addresses, chain_info) + .with_limits(start_height, max_entries), } } } @@ -3904,14 +3952,22 @@ impl From for GetAddressUtxosRequest { enum DGetAddressUtxosRequest { /// A single address string. Single(String), - /// A full request object with address list and chainInfo flag. + /// A full request object with an address list, a chainInfo flag, and optional limits. Object { - /// A list of addresses to get transactions from. + /// A list of addresses to get unspent outputs from. addresses: Vec, - /// The height to start looking for transactions. + /// Whether to return chain info along with the unspent outputs. #[serde(default)] #[serde(rename = "chainInfo")] chain_info: bool, + /// The height to start looking for unspent outputs, or zero for the whole chain. + #[serde(default)] + #[serde(rename = "startHeight")] + start_height: u64, + /// The maximum number of unspent outputs to return, or zero for no limit. + #[serde(default)] + #[serde(rename = "maxEntries")] + max_entries: u32, }, } diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 983a48999f2..85c8005b746 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1963,6 +1963,95 @@ async fn rpc_getaddressutxos_response() { mempool.expect_no_requests().await; } +/// Checks that `startHeight` and `maxEntries` select the same UTXOs that filtering the full +/// response would have, now that the state applies them to the index scan instead. +#[tokio::test(flavor = "multi_thread")] +async fn rpc_getaddressutxos_limits() { + let _init_guard = zebra_test::init(); + + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .values() + .map(|block_bytes| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // The address that receives the second output of every coinbase transaction, + // which is always `t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd`. + let address = &blocks[1].transactions[0].outputs()[1] + .address(&Mainnet) + .unwrap(); + + let mut mempool: MockService<_, _, _, BoxError> = MockService::build().for_unit_tests(); + let (state, read_state, tip, _) = zebra_state::populated_state(blocks.clone(), &Mainnet).await; + + let (_tx, rx) = tokio::sync::watch::channel(None); + let (rpc, _) = RpcImpl::new( + Mainnet, + Default::default(), + Default::default(), + "0.0.1", + "RPC test", + Buffer::new(mempool.clone(), 1), + state.clone(), + Buffer::new(read_state.clone(), 1), + MockService::build().for_unit_tests(), + MockSyncStatus::default(), + tip, + MockAddressBookPeers::default(), + rx, + None, + ); + + let addresses = vec![address.to_string()]; + + macro_rules! utxos { + ($start_height:expr, $max_entries:expr) => {{ + let response = rpc + .get_address_utxos( + GetAddressUtxosRequest::new(addresses.clone(), false) + .with_limits($start_height, $max_entries), + ) + .await + .expect("address is valid so no error can happen here"); + + let GetAddressUtxosResponse::Utxos(utxos) = response else { + panic!("expected GetAddressUtxosResponse::ChainInfoFalse variant"); + }; + + utxos + }}; + } + + // Unset limits return everything, exactly as they did before the limits existed. + let all = utxos!(0, 0); + assert_eq!(all.len(), 10); + + // A limit takes the first entries in chain order, not an arbitrary subset. + assert_eq!(utxos!(0, 3), all[..3].to_vec()); + + // A start height drops the entries below it, and keeps the rest in the same order. + let start_height = all[5].height().0; + let from_start_height: Vec<_> = all + .iter() + .filter(|utxo| utxo.height().0 >= start_height) + .cloned() + .collect(); + assert_eq!(utxos!(u64::from(start_height), 0), from_start_height); + + // Together, they take the first entries at or above the start height. A backend that + // truncated before filtering by height would return the entries below it instead. + assert_eq!( + utxos!(u64::from(start_height), 2), + from_start_height[..2].to_vec() + ); + + // A start height past the tip selects nothing, and one past the maximum block height is + // clamped rather than rejected. + assert!(utxos!(u64::from(u32::MAX), 0).is_empty()); + assert!(utxos!(u64::from(u32::MAX) + 1, 0).is_empty()); + + mempool.expect_no_requests().await; +} + #[tokio::test(flavor = "multi_thread")] async fn rpc_getblockcount() { let _init_guard = zebra_test::init();