From caae082a6aa285f1f890c9884f896ce6b23b9828 Mon Sep 17 00:00:00 2001 From: Mykhailo Kremniov Date: Thu, 18 Sep 2025 14:14:46 +0300 Subject: [PATCH] Implement chainstate dumper tool --- Cargo.lock | 25 + Cargo.toml | 2 + chainstate/db-dumper/Cargo.toml | 39 ++ chainstate/db-dumper/src/dumper/main.rs | 61 ++ chainstate/db-dumper/src/dumper/options.rs | 117 ++++ .../db-dumper/src/dumper_lib/dump_blocks.rs | 258 +++++++++ chainstate/db-dumper/src/dumper_lib/error.rs | 67 +++ chainstate/db-dumper/src/dumper_lib/fields.rs | 64 +++ chainstate/db-dumper/src/dumper_lib/lib.rs | 28 + .../db-dumper/src/dumper_lib/tests/mod.rs | 534 ++++++++++++++++++ chainstate/db-dumper/src/dumper_lib/utils.rs | 95 ++++ chainstate/launcher/src/lib.rs | 4 +- chainstate/src/detail/bootstrap.rs | 4 +- .../src/interface/chainstate_interface.rs | 7 +- .../interface/chainstate_interface_impl.rs | 4 +- .../chainstate_interface_impl_delegation.rs | 4 +- chainstate/src/rpc/mod.rs | 6 +- mocks/src/chainstate.rs | 2 +- node-daemon/docs/RPC.md | 2 +- node-lib/src/options.rs | 4 +- storage/lmdb/src/lib.rs | 37 +- 21 files changed, 1340 insertions(+), 24 deletions(-) create mode 100644 chainstate/db-dumper/Cargo.toml create mode 100644 chainstate/db-dumper/src/dumper/main.rs create mode 100644 chainstate/db-dumper/src/dumper/options.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/dump_blocks.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/error.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/fields.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/lib.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/tests/mod.rs create mode 100644 chainstate/db-dumper/src/dumper_lib/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 835a1bb309..fc2e61cc73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1294,6 +1294,31 @@ dependencies = [ "utxo", ] +[[package]] +name = "chainstate-db-dumper" +version = "1.1.0" +dependencies = [ + "anyhow", + "chainstate", + "chainstate-launcher", + "chainstate-storage", + "chainstate-types", + "clap", + "common", + "crypto", + "ctor", + "hex", + "itertools 0.14.0", + "logging", + "mocks", + "rstest", + "storage-lmdb", + "strum", + "test-utils", + "thiserror 1.0.69", + "utils", +] + [[package]] name = "chainstate-launcher" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 6bd181da2e..aea5961e00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "api-server/web-server", # API server, for light-wallets and block explorers: web-server. "blockprod", # Block production with whatever consensus algorithm. "chainstate", # Code on chainstate of blocks and transactions. + "chainstate/db-dumper", # A tool for dumping the contents of the chainstate db. "chainstate/test-suite", # Tests for the chainstate, separated to make use of the chainstate test framework. "common", # Everything else, until it's moved to another crate. "consensus", # Consensus related logic. @@ -86,6 +87,7 @@ default-members = [ "api-server/scanner-daemon", "api-server/web-server", "chainstate", + "chainstate/db-dumper", "common", "crypto", "dns-server", diff --git a/chainstate/db-dumper/Cargo.toml b/chainstate/db-dumper/Cargo.toml new file mode 100644 index 0000000000..9c0d305df2 --- /dev/null +++ b/chainstate/db-dumper/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "chainstate-db-dumper" +license.workspace = true +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[lib] +name = "chainstate_db_dumper_lib" +path = "src/dumper_lib/lib.rs" + +[dependencies] +chainstate = { path = ".." } +chainstate-launcher = { path = "../launcher" } +chainstate-storage = { path = "../storage" } +chainstate-types = { path = "../types" } +common = { path = "../../common" } +logging = { path = "../../logging" } +storage-lmdb = { path = "../../storage/lmdb" } +utils = { path = "../../utils" } + +anyhow.workspace = true +clap.workspace = true +itertools.workspace = true +strum.workspace = true +thiserror.workspace = true + +[dev-dependencies] +crypto = { path = "../../crypto" } +mocks = { path = "../../mocks" } +test-utils = { path = "../../test-utils" } + +ctor.workspace = true +hex.workspace = true +rstest.workspace = true + +[[bin]] +name = "chainstate-db-dumper" +path = "src/dumper/main.rs" diff --git a/chainstate/db-dumper/src/dumper/main.rs b/chainstate/db-dumper/src/dumper/main.rs new file mode 100644 index 0000000000..2a6b547f92 --- /dev/null +++ b/chainstate/db-dumper/src/dumper/main.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use chainstate_launcher::SUBDIRECTORY_LMDB; +use logging::{init_logging, log}; +use utils::default_data_dir::default_data_dir_for_chain; + +use chainstate_db_dumper_lib::{dump_blocks_to_file, parse_block_output_fields_list}; + +use crate::options::{default_fields, Options}; + +mod options; + +fn run() -> anyhow::Result<()> { + let opts = Options::parse(); + let chain_type = opts.chain_type.chain_type(); + let db_dir = opts + .db_dir + .unwrap_or_else(|| default_data_dir_for_chain(chain_type.name()).join(SUBDIRECTORY_LMDB)); + let fields = opts.fields.map(|fields| parse_block_output_fields_list(&fields)).transpose()?; + let fields = fields.as_deref().unwrap_or(default_fields(opts.mainchain_only)); + + log::info!("Using db dir {}", db_dir.display()); + + dump_blocks_to_file( + chain_type, + db_dir, + opts.mainchain_only, + opts.from_height, + fields, + &opts.output_file, + )?; + Ok(()) +} + +fn main() { + utils::rust_backtrace::enable(); + + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "info"); + } + + init_logging(); + + run().unwrap_or_else(|err| { + eprintln!("Error: {err:?}"); + std::process::exit(1) + }) +} diff --git a/chainstate/db-dumper/src/dumper/options.rs b/chainstate/db-dumper/src/dumper/options.rs new file mode 100644 index 0000000000..761e42d40f --- /dev/null +++ b/chainstate/db-dumper/src/dumper/options.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::PathBuf; + +use clap::FromArgMatches as _; +use itertools::Itertools as _; +use strum::IntoEnumIterator as _; + +use common::chain::config::ChainType; + +use chainstate_db_dumper_lib::{ + BlockOutputField, DEFAULT_BLOCK_OUTPUT_FIELDS_MAINCHAIN_ONLY, + DEFAULT_BLOCK_OUTPUT_FIELDS_WITH_STALE_CHAINS, +}; + +#[derive(clap::ValueEnum, Debug, Clone)] +pub enum ChainTypeOption { + Mainnet, + Testnet, + Regtest, + Signet, +} + +impl ChainTypeOption { + pub fn chain_type(&self) -> ChainType { + match self { + ChainTypeOption::Mainnet => ChainType::Mainnet, + ChainTypeOption::Testnet => ChainType::Testnet, + ChainTypeOption::Regtest => ChainType::Regtest, + ChainTypeOption::Signet => ChainType::Signet, + } + } +} + +const MAINCHAIN_ONLY_OPT_NAME: &str = "mainchain-only"; + +/// Dump block information from the chainstate db into a CSV file +#[derive(clap::Parser, Debug, Clone)] +pub struct Options { + /// Chain type + #[clap(short, long = "chain-type")] + pub chain_type: ChainTypeOption, + + /// The path to the chainstate-lmdb directory. + /// + /// The default value is the default location corresponding to the specified chain type. + #[clap(short, long = "db-dir")] + pub db_dir: Option, + + /// Output file + #[clap(short, long = "output-file")] + pub output_file: PathBuf, + + /// Whether to only dump mainchain blocks + #[clap(long = MAINCHAIN_ONLY_OPT_NAME, action = clap::ArgAction::Set, default_value_t = true)] + pub mainchain_only: bool, + + /// Block height to start from + #[clap(long = "from_height", default_value_t = 0)] + pub from_height: u64, + + /// This help string + #[clap(long = "fields")] + pub fields: Option, +} + +impl Options { + /// Build the command adding custom description to "fields". + pub fn build() -> clap::Command { + let default_fields_mc_only = default_fields(true).iter().join(","); + let default_fields_all_blocks = default_fields(false).iter().join(","); + let all_fields = BlockOutputField::iter().join(", "); + + let fields_help = format!( + concat!( + "Comma-separated list of fields to dump.\n", + "The default value depends on --{}, if true: '{}', if false: '{}'\n", + "All possible fields are: {}" + ), + MAINCHAIN_ONLY_OPT_NAME, default_fields_mc_only, default_fields_all_blocks, all_fields + ); + + let cmd = ::command(); + cmd.mut_arg("fields", |arg| arg.help(fields_help)) + } + + /// Custom `parse` function that used `build` defined above. + pub fn parse() -> Self { + let matches = Self::build().get_matches(); + + match Self::from_arg_matches(&matches) { + Ok(this) => this, + Err(err) => err.exit(), + } + } +} + +pub fn default_fields(mainchain_only: bool) -> &'static [BlockOutputField] { + if mainchain_only { + &DEFAULT_BLOCK_OUTPUT_FIELDS_MAINCHAIN_ONLY + } else { + &DEFAULT_BLOCK_OUTPUT_FIELDS_WITH_STALE_CHAINS + } +} diff --git a/chainstate/db-dumper/src/dumper_lib/dump_blocks.rs b/chainstate/db-dumper/src/dumper_lib/dump_blocks.rs new file mode 100644 index 0000000000..694f43edd3 --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/dump_blocks.rs @@ -0,0 +1,258 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use chainstate::{chainstate_interface::ChainstateInterface, BlockIndex}; +use chainstate_types::{BlockStatus, BlockValidationStage}; +use common::{ + address::Address, + chain::{self, config::ChainType, ChainConfig, Genesis}, + primitives::id::WithId, + Uint256, +}; +use logging::log; +use utils::ensure; + +use crate::{ + utils::{ + create_chainstate, get_pos_consensus_data, map_output_write_err, write_comma_if_needed, + write_eol, + }, + BlockOutputField, Error, +}; + +// Note: this function will fail if the genesis id in the created chain config doesn't +// match the inferred one (i.e. the parent of the first block, this check is done inside +// Chainstate, search for the `GenesisMismatch` error). This means that Regtest/Signet chains +// with custom geneses are not supported ATM. +// TODO: perhaps we need a way to disable the genesis check in the chainstate. +pub fn dump_blocks_to_file( + chain_type: ChainType, + db_path: PathBuf, + mainchain_only: bool, + from_height: u64, + fields: &[BlockOutputField], + file_path: &Path, +) -> Result<(), Error> { + let output = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(file_path) + .map_err(|err| Error::CannotOpenOutputFile { + error: err.to_string(), + })?; + // Note: unbuffered output is very slow. + let mut output = std::io::BufWriter::new(output); + + let chain_config = Arc::new(chain::config::Builder::new(chain_type).build()); + let chainstate = create_chainstate(chain_config, db_path)?; + + dump_blocks_generic( + &chainstate, + mainchain_only, + from_height, + fields, + &mut output, + ) +} + +pub fn dump_blocks_generic( + chainstate: &dyn ChainstateInterface, + mainchain_only: bool, + from_height: u64, + fields: &[BlockOutputField], + output: &mut impl std::io::Write, +) -> Result<(), Error> { + if mainchain_only { + log::info!("Dumping mainchain blocks only"); + } else { + log::info!("Dumping all blocks"); + } + + let chain_config = chainstate.get_chain_config(); + + let blocks_ids = if mainchain_only { + chainstate.get_mainchain_blocks_list()? + } else { + chainstate.get_block_id_tree_as_list()? + }; + + write_header(fields, output)?; + write_eol(output)?; + + if from_height == 0 { + write_genesis(chain_config.genesis_block(), fields, output)?; + write_eol(output)?; + } + + let mut prev_block_height = 0; + for (block_id_idx, block_id) in blocks_ids.iter().enumerate() { + let block_index = chainstate + .get_block_index_for_any_block(block_id)? + .ok_or(Error::BlockIndexNotFound(*block_id))?; + + let cur_block_height = block_index.block_height().into_int(); + ensure!( + cur_block_height == prev_block_height || cur_block_height == prev_block_height + 1, + Error::UnexpectedBlockOrder { + cur_block_height, + prev_block_height + } + ); + + if cur_block_height >= from_height { + let is_mainchain = + mainchain_only || chainstate.is_block_in_main_chain(block_id.into())?; + let is_mainchain = if is_mainchain { + IsMainchain::Yes + } else { + IsMainchain::No + }; + write_block(&block_index, is_mainchain, fields, chain_config, output)?; + + if block_id_idx + 1 != blocks_ids.len() { + write_eol(output)?; + } + } + + prev_block_height = cur_block_height; + } + + Ok(()) +} + +#[derive(Eq, PartialEq, Debug, Copy, Clone, strum::Display)] +pub enum IsMainchain { + #[strum(serialize = "y")] + Yes, + + #[strum(serialize = "n")] + No, +} + +#[derive(Eq, PartialEq, Debug, Copy, Clone, strum::Display, strum::EnumIter)] +pub enum BlockStatusOutput { + #[strum(serialize = "b")] + Bad, + + #[strum(serialize = "u")] + Unchecked, + + #[strum(serialize = "p")] + PartiallyChecked, + + #[strum(serialize = "g")] + Good, +} + +impl BlockStatusOutput { + fn from(status: BlockStatus) -> Self { + if !status.is_ok() { + Self::Bad + } else { + match status.last_valid_stage() { + BlockValidationStage::Unchecked => Self::Unchecked, + BlockValidationStage::CheckBlockOk => Self::PartiallyChecked, + BlockValidationStage::FullyChecked => Self::Good, + } + } + } +} + +fn write_header( + fields: &[BlockOutputField], + output: &mut impl std::io::Write, +) -> Result<(), Error> { + for (idx, field) in fields.iter().enumerate() { + write!(output, "{field}").map_err(map_output_write_err)?; + write_comma_if_needed(idx, fields, output)?; + } + + Ok(()) +} + +fn write_genesis( + genesis: &WithId, + fields: &[BlockOutputField], + output: &mut impl std::io::Write, +) -> Result<(), Error> { + for (idx, field) in fields.iter().enumerate() { + match field { + BlockOutputField::Height => write!(output, "0"), + BlockOutputField::IsMainchain => write!(output, "{}", IsMainchain::Yes), + BlockOutputField::Id => write!(output, "{:x}", WithId::id(genesis)), + BlockOutputField::Timestamp => { + write!(output, "{}", genesis.timestamp().as_int_seconds()) + } + BlockOutputField::Status => write!(output, "{}", BlockStatusOutput::Good), + BlockOutputField::PoolId + | BlockOutputField::Target + | BlockOutputField::ChainTrust + | BlockOutputField::ParentId => write!(output, "-"), + } + .map_err(map_output_write_err)?; + + write_comma_if_needed(idx, fields, output)?; + } + + Ok(()) +} + +fn write_block( + block_index: &BlockIndex, + is_mainchain: IsMainchain, + fields: &[BlockOutputField], + chain_config: &ChainConfig, + output: &mut impl std::io::Write, +) -> Result<(), Error> { + for (idx, field) in fields.iter().enumerate() { + match field { + BlockOutputField::Height => write!(output, "{}", block_index.block_height().into_int()), + BlockOutputField::IsMainchain => write!(output, "{is_mainchain}"), + BlockOutputField::Id => write!(output, "{:x}", block_index.block_id()), + BlockOutputField::Timestamp => { + write!(output, "{}", block_index.block_timestamp().as_int_seconds()) + } + BlockOutputField::Status => { + write!(output, "{}", BlockStatusOutput::from(block_index.status())) + } + BlockOutputField::PoolId => { + let pool_id = get_pos_consensus_data(block_index)?.stake_pool_id(); + let pool_id_str = Address::new(chain_config, *pool_id) + .map_err(Error::AddressConstructionError)? + .into_string(); + write!(output, "{pool_id_str}") + } + BlockOutputField::Target => { + let compact_target = get_pos_consensus_data(block_index)?.compact_target(); + let target = Uint256::try_from(compact_target) + .map_err(|_| Error::BlockCompactTargetUnpackingError(compact_target))?; + write!(output, "{target:x}") + } + BlockOutputField::ChainTrust => write!(output, "{:x}", block_index.chain_trust()), + BlockOutputField::ParentId => write!(output, "{:x}", block_index.prev_block_id()), + } + .map_err(map_output_write_err)?; + + write_comma_if_needed(idx, fields, output)?; + } + + Ok(()) +} diff --git a/chainstate/db-dumper/src/dumper_lib/error.rs b/chainstate/db-dumper/src/dumper_lib/error.rs new file mode 100644 index 0000000000..373a9191b4 --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/error.rs @@ -0,0 +1,67 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use chainstate::{ChainstateError, StorageCompatibilityCheckError}; +use common::{ + address::AddressError, + chain::Block, + primitives::{Compact, Id}, +}; + +#[derive(thiserror::Error, Clone, Debug)] +pub enum Error { + #[error(transparent)] + ChainstateError(#[from] ChainstateError), + + #[error("Storage creation error: {0}")] + StorageCreationError(chainstate_storage::Error), + + #[error("Storage compatibility check error: {0}")] + StorageCompatibilityCheckError(#[from] StorageCompatibilityCheckError), + + #[error("No block ids returned")] + NoBlockIdsReturned, + + #[error("Block index not found for block {0:x}")] + BlockIndexNotFound(Id), + + #[error("Error writing to the output file: {0}")] + OutputWriteError(String), + + #[error("Non-PoS consensus type in block {0:x}")] + NonPoSConsensusInBlock(Id), + + #[error("Address construction error")] + AddressConstructionError(AddressError), + + #[error("Error unpacking compact target {0:?} to Uint256")] + BlockCompactTargetUnpackingError(Compact), + + #[error( + "Obtained blocks are in unexpected order, current height is {}, previous height is {}", + cur_block_height, + prev_block_height + )] + UnexpectedBlockOrder { + cur_block_height: u64, + prev_block_height: u64, + }, + + #[error("Cannot open output file: {error}")] + CannotOpenOutputFile { error: String }, + + #[error("Unexpected output field: {field}")] + UnexpectedOutputField { field: String }, +} diff --git a/chainstate/db-dumper/src/dumper_lib/fields.rs b/chainstate/db-dumper/src/dumper_lib/fields.rs new file mode 100644 index 0000000000..ca1b0818cf --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/fields.rs @@ -0,0 +1,64 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::Error; + +// Note: the order of items of this enum is how they will appear in the help message. +// Though not super important, it's nicer when the order is consistent with the contents +// of the "DEFAULT_BLOCK_OUTPUT_FIELDS_XXX" arrays. +#[derive(Eq, PartialEq, Debug, Copy, Clone, strum::Display, strum::EnumString, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum BlockOutputField { + Height, + IsMainchain, + Id, + Timestamp, + PoolId, + Target, + ChainTrust, + Status, + ParentId, +} + +pub static DEFAULT_BLOCK_OUTPUT_FIELDS_MAINCHAIN_ONLY: [BlockOutputField; 5] = [ + BlockOutputField::Height, + BlockOutputField::Id, + BlockOutputField::Timestamp, + BlockOutputField::PoolId, + BlockOutputField::Target, +]; + +pub static DEFAULT_BLOCK_OUTPUT_FIELDS_WITH_STALE_CHAINS: [BlockOutputField; 7] = [ + BlockOutputField::Height, + BlockOutputField::IsMainchain, + BlockOutputField::Id, + BlockOutputField::Timestamp, + BlockOutputField::PoolId, + BlockOutputField::Target, + BlockOutputField::ParentId, +]; + +pub fn parse_block_output_fields_list(list: &str) -> Result, Error> { + let result = list + .split(',') + .map(|field| { + let field = field.trim(); + field.parse::().map_err(|_| Error::UnexpectedOutputField { + field: field.to_owned(), + }) + }) + .collect::, _>>()?; + Ok(result) +} diff --git a/chainstate/db-dumper/src/dumper_lib/lib.rs b/chainstate/db-dumper/src/dumper_lib/lib.rs new file mode 100644 index 0000000000..fc66c5591a --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/lib.rs @@ -0,0 +1,28 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod dump_blocks; +mod error; +mod fields; +#[cfg(test)] +mod tests; +mod utils; + +pub use dump_blocks::{dump_blocks_generic, dump_blocks_to_file}; +pub use error::Error; +pub use fields::{ + parse_block_output_fields_list, BlockOutputField, DEFAULT_BLOCK_OUTPUT_FIELDS_MAINCHAIN_ONLY, + DEFAULT_BLOCK_OUTPUT_FIELDS_WITH_STALE_CHAINS, +}; diff --git a/chainstate/db-dumper/src/dumper_lib/tests/mod.rs b/chainstate/db-dumper/src/dumper_lib/tests/mod.rs new file mode 100644 index 0000000000..6e09d2fea4 --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/tests/mod.rs @@ -0,0 +1,534 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{collections::BTreeMap, str::FromStr as _, sync::Arc}; + +use itertools::Itertools as _; +use rstest::rstest; +use strum::IntoEnumIterator as _; + +use chainstate::BlockIndex; +use chainstate_launcher::ChainConfig; +use chainstate_types::{vrf_tools::construct_transcript, BlockStatus, BlockValidationStage}; +use common::{ + address::Address, + chain::{ + self, + block::{consensus_data::PoSData, timestamp::BlockTimestamp, BlockReward, ConsensusData}, + Block, GenBlock, Genesis, PoolId, + }, + primitives::{BlockHeight, Compact, Id, H256}, + Uint256, +}; +use crypto::vrf::{VRFKeyKind, VRFPrivateKey, VRFReturn}; +use mocks::MockChainstateInterface; +use test_utils::{ + random::{ + make_seedable_rng, randomness::SliceRandom, CryptoRng, IteratorRandom as _, Rng, Seed, + }, + random_ascii_alphanumeric_string, +}; + +use crate::{dump_blocks::BlockStatusOutput, dump_blocks_generic, BlockOutputField}; + +#[ctor::ctor] +fn init() { + logging::init_logging(); +} + +#[test] +fn dump_blocks_predefined() { + let genesis = Genesis::new( + "foo".to_owned(), + BlockTimestamp::from_int_seconds(12345), + vec![], + ); + let chain_config = + Arc::new(chain::config::create_unit_test_config_builder().genesis_custom(genesis).build()); + + let block_infos = vec![ + TestBlockInfo::from_input_info(TestBlockInputInfo { + height: BlockHeight::new(1), + is_mainchain: true, + parent_id: id_from_str( + "1111111111111111111111111111111111111111111111111111111111111111", + ), + timestamp: BlockTimestamp::from_int_seconds(123), + pool_id: pool_id_from_str( + "rpool1hd38tvxv3em8wazcvaxhg2fm3r2k9lt69azdcceagsgaa97x4hcqlfytgj", + &chain_config, + ), + target: uint256_from_str( + "1111110000000000000000000000000000000000000000000000000000000000", + ), + chain_trust: uint256_from_str( + "2222222222222222222222222222222333333333333333333333333333333333", + ), + status: BlockStatusOutput::Good, + }), + TestBlockInfo::from_input_info(TestBlockInputInfo { + height: BlockHeight::new(2), + is_mainchain: true, + parent_id: id_from_str( + "2222222222222222222222222222222222222222222222222222222222222222", + ), + timestamp: BlockTimestamp::from_int_seconds(234), + pool_id: pool_id_from_str( + "rpool19r5wd2yyr4cdjc4lrhhwey0j47959xq9quf0vxrqqhd984zuhtps87u6an", + &chain_config, + ), + target: uint256_from_str( + "2222220000000000000000000000000000000000000000000000000000000000", + ), + chain_trust: uint256_from_str( + "3333333333333333333333333333333333444444444444444444444444444444", + ), + status: BlockStatusOutput::PartiallyChecked, + }), + TestBlockInfo::from_input_info(TestBlockInputInfo { + height: BlockHeight::new(2), + is_mainchain: false, + parent_id: id_from_str( + "3333333333333333333333333333333333333333333333333333333333333333", + ), + timestamp: BlockTimestamp::from_int_seconds(345), + pool_id: pool_id_from_str( + "rpool1jxrvvujqm4plkr7rfshmru8slddw057npf5fwv7t07awtta6ccaq4c9ycx", + &chain_config, + ), + target: uint256_from_str( + "3333330000000000000000000000000000000000000000000000000000000000", + ), + chain_trust: uint256_from_str( + "4444444444444444444444444444444444455555555555555555555555555555", + ), + status: BlockStatusOutput::Unchecked, + }), + TestBlockInfo::from_input_info(TestBlockInputInfo { + height: BlockHeight::new(3), + is_mainchain: true, + parent_id: id_from_str( + "4444444444444444444444444444444444444444444444444444444444444444", + ), + timestamp: BlockTimestamp::from_int_seconds(456), + pool_id: pool_id_from_str( + "rpool1gjlw2v8nmr78tcxxp70gp0jnplhkwwyhem720puqa4zhkven0cdsfrrlka", + &chain_config, + ), + target: uint256_from_str( + "4444440000000000000000000000000000000000000000000000000000000000", + ), + chain_trust: uint256_from_str( + "5555555555555555555555555555555555566666666666666666666666666666", + ), + status: BlockStatusOutput::Bad, + }), + ]; + + let block_infos_by_id = Arc::new( + block_infos + .iter() + .map(|info| ((*info.block_index.block_id()).into(), info.clone())) + .collect::, _>>(), + ); + let all_block_ids_in_order = + block_infos.iter().map(|info| *info.block_index.block_id()).collect_vec(); + + let mut chainstate = MockChainstateInterface::new(); + + chainstate.expect_get_chain_config().return_const(Arc::clone(&chain_config)); + + chainstate + .expect_get_block_id_tree_as_list() + .returning(move || Ok(all_block_ids_in_order.clone())); + + chainstate.expect_get_block_index_for_any_block().returning({ + let block_infos_by_id = Arc::clone(&block_infos_by_id); + move |block_id| { + Ok(Some( + block_infos_by_id.get(block_id.into()).unwrap().block_index.clone(), + )) + } + }); + + chainstate.expect_is_block_in_main_chain().returning({ + let block_infos_by_id = Arc::clone(&block_infos_by_id); + move |block_id| Ok(block_infos_by_id.get(block_id).unwrap().input_info.is_mainchain) + }); + + // Check all fields + { + let output_lines = { + let mut output = Vec::::new(); + + dump_blocks_generic( + &chainstate, + false, + 0, + &BlockOutputField::iter().collect_vec(), + &mut output, + ) + .unwrap(); + + String::from_utf8(output).unwrap().lines().map(ToOwned::to_owned).collect_vec() + }; + + let expected_output_lines = vec![ + "height,is_mainchain,id,timestamp,pool_id,target,chain_trust,status,parent_id".to_owned(), + "0,y,ac50d72a82f0dad2033ea2d2e36fdab486cdb8d5088d1d7eb7d85c71c05d5e5d,12345,-,-,-,g,-".to_owned(), + "1,y,2f1b62916aa3fc731b27fec1ddbfca06b75715d68f51bcbc6f3107714cb69a71,123,rpool1hd38tvxv3em8wazcvaxhg2fm3r2k9lt69azdcceagsgaa97x4hcqlfytgj,1111110000000000000000000000000000000000000000000000000000000000,2222222222222222222222222222222333333333333333333333333333333333,g,1111111111111111111111111111111111111111111111111111111111111111".to_owned(), + "2,y,96d754b644e350312850aac926a5b290809d8084fd63de765036c44054c1c8b6,234,rpool19r5wd2yyr4cdjc4lrhhwey0j47959xq9quf0vxrqqhd984zuhtps87u6an,2222220000000000000000000000000000000000000000000000000000000000,3333333333333333333333333333333333444444444444444444444444444444,p,2222222222222222222222222222222222222222222222222222222222222222".to_owned(), + "2,n,a7b5752f6d3aceeaef461f1171b7b6a39f8c4c30e4c2e7e5bab243bb001ea411,345,rpool1jxrvvujqm4plkr7rfshmru8slddw057npf5fwv7t07awtta6ccaq4c9ycx,3333330000000000000000000000000000000000000000000000000000000000,4444444444444444444444444444444444455555555555555555555555555555,u,3333333333333333333333333333333333333333333333333333333333333333".to_owned(), + "3,y,dc863d845e864a902986a1fcb813a6346c83ebc8198db5f710759e00f590f86f,456,rpool1gjlw2v8nmr78tcxxp70gp0jnplhkwwyhem720puqa4zhkven0cdsfrrlka,4444440000000000000000000000000000000000000000000000000000000000,5555555555555555555555555555555555566666666666666666666666666666,b,4444444444444444444444444444444444444444444444444444444444444444".to_owned() + ]; + assert_eq!(output_lines, expected_output_lines); + } + + // Check some of the fields + { + let output_lines = { + let mut output = Vec::::new(); + + dump_blocks_generic( + &chainstate, + false, + 0, + &[ + // Note: IsMainchain and Height are swapped compared to the "default" order. + BlockOutputField::IsMainchain, + BlockOutputField::Height, + BlockOutputField::Timestamp, + ], + &mut output, + ) + .unwrap(); + + String::from_utf8(output).unwrap().lines().map(ToOwned::to_owned).collect_vec() + }; + + let expected_output_lines = vec![ + "is_mainchain,height,timestamp".to_owned(), + "y,0,12345".to_owned(), + "y,1,123".to_owned(), + "y,2,234".to_owned(), + "n,2,345".to_owned(), + "y,3,456".to_owned(), + ]; + assert_eq!(output_lines, expected_output_lines); + } +} + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +fn dump_blocks_random( + #[case] seed: Seed, + #[values(false, true)] mainchain_only: bool, + #[values(false, true)] start_from_zero_height: bool, +) { + let mut rng = make_seedable_rng(seed); + + let genesis_msg = random_ascii_alphanumeric_string(&mut rng, 10..20); + let genesis_timestamp = BlockTimestamp::from_int_seconds(rng.gen_range(0..1_000_000)); + let genesis = Genesis::new(genesis_msg, genesis_timestamp, vec![]); + let chain_config = + Arc::new(chain::config::create_unit_test_config_builder().genesis_custom(genesis).build()); + + let mainchain_block_count = rng.gen_range(10..20); + let stale_block_count = rng.gen_range(0..mainchain_block_count); + + let block_infos = { + let mut infos = Vec::new(); + + for i in 0..mainchain_block_count { + let height = BlockHeight::new(i + 1); + let input_info = TestBlockInputInfo::from_rng(height, true, &mut rng); + infos.push(TestBlockInfo::from_input_info(input_info)); + } + + if !mainchain_only { + for _ in 0..stale_block_count { + let height = BlockHeight::new(rng.gen_range(1..=mainchain_block_count)); + let input_info = TestBlockInputInfo::from_rng(height, false, &mut rng); + infos.push(TestBlockInfo::from_input_info(input_info)); + } + + infos.shuffle(&mut rng); + infos.sort_by(|b1, b2| b1.input_info.height.cmp(&b2.input_info.height)); + } + + infos + }; + let block_infos_by_id = Arc::new( + block_infos + .iter() + .map(|info| ((*info.block_index.block_id()).into(), info.clone())) + .collect::, _>>(), + ); + let all_block_ids_in_order = + block_infos.iter().map(|info| *info.block_index.block_id()).collect_vec(); + + let mut chainstate = MockChainstateInterface::new(); + + chainstate.expect_get_chain_config().return_const(Arc::clone(&chain_config)); + + if mainchain_only { + chainstate + .expect_get_mainchain_blocks_list() + .returning(move || Ok(all_block_ids_in_order.clone())); + } else { + chainstate + .expect_get_block_id_tree_as_list() + .returning(move || Ok(all_block_ids_in_order.clone())); + } + + chainstate.expect_get_block_index_for_any_block().returning({ + let block_infos_by_id = Arc::clone(&block_infos_by_id); + move |block_id| { + Ok(Some( + block_infos_by_id.get(block_id.into()).unwrap().block_index.clone(), + )) + } + }); + if !mainchain_only { + chainstate.expect_is_block_in_main_chain().returning({ + let block_infos_by_id = Arc::clone(&block_infos_by_id); + move |block_id| Ok(block_infos_by_id.get(block_id).unwrap().input_info.is_mainchain) + }); + } + + let start_height = if start_from_zero_height { + 0 + } else { + rng.gen_range(1..=mainchain_block_count) + }; + + let starting_block_info_index = block_infos + .iter() + .position(|info| info.input_info.height.into_int() >= start_height) + .unwrap(); + + let output_lines = { + let mut output = Vec::::new(); + + let fields = BlockOutputField::iter().collect_vec(); + dump_blocks_generic( + &chainstate, + mainchain_only, + start_height, + &fields, + &mut output, + ) + .unwrap(); + + String::from_utf8(output).unwrap().lines().map(ToOwned::to_owned).collect_vec() + }; + + let expected_output_lines = { + let mut lines = Vec::new(); + lines.push(expected_header_for_default_field_order().to_owned()); + if start_from_zero_height { + lines.push(expected_genesis_output_line_for_default_field_order( + &chain_config, + )); + } + for info in &block_infos[starting_block_info_index..] { + lines.push(expected_output_line_for_default_field_order( + info, + &chain_config, + )); + } + + lines + }; + + assert_eq!(output_lines, expected_output_lines); +} + +#[derive(Clone)] +struct TestBlockInputInfo { + // Note: the only thing the dumper checks is that the returned blocks are ordered by height + // and there are no gaps between them. So, these two fields cannot be absolutely arbitrary. + height: BlockHeight, + is_mainchain: bool, + + // But the rest of them can. + parent_id: Id, + timestamp: BlockTimestamp, + pool_id: PoolId, + target: Uint256, + chain_trust: Uint256, + status: BlockStatusOutput, + // Also note that we don't have the block id here, this is because BlockIndex (which we + // need to return from chainstate) only accepts an entire block and calculates the id + // on its own. +} + +impl TestBlockInputInfo { + fn from_rng( + height: BlockHeight, + is_mainchain: bool, + rng: &mut (impl Rng + CryptoRng), + ) -> TestBlockInputInfo { + Self { + height, + is_mainchain, + parent_id: Id::random_using(rng), + timestamp: BlockTimestamp::from_int_seconds(rng.gen()), + pool_id: PoolId::random_using(rng), + target: gen_target(rng), + chain_trust: Uint256::from_bytes(rng.gen()), + status: BlockStatusOutput::iter().choose(rng).unwrap(), + } + } +} + +#[derive(Clone)] +struct TestBlockInfo { + input_info: TestBlockInputInfo, + // The BlockIndex that contains the input info as well as the block id. + block_index: BlockIndex, +} + +impl TestBlockInfo { + fn from_input_info(input_info: TestBlockInputInfo) -> Self { + let block = Block::new( + vec![], + input_info.parent_id, + input_info.timestamp, + make_consensus_data(input_info.pool_id, input_info.target.into()), + BlockReward::new(vec![]), + ) + .unwrap(); + + let block_status = match input_info.status { + BlockStatusOutput::Bad => bad_block_status(), + BlockStatusOutput::Unchecked => { + BlockStatus::new_at_stage(BlockValidationStage::Unchecked) + } + BlockStatusOutput::PartiallyChecked => { + BlockStatus::new_at_stage(BlockValidationStage::CheckBlockOk) + } + BlockStatusOutput::Good => { + BlockStatus::new_at_stage(BlockValidationStage::FullyChecked) + } + }; + let block_index = BlockIndex::new( + &block, + input_info.chain_trust, + // some_ancestor - doesn't matter + Id::zero(), + input_info.height, + // chain_time_max - doesn't matter + BlockTimestamp::from_int_seconds(0), + // chain_transaction_count - doesn't matter + 0, + block_status, + ); + + Self { + input_info, + block_index, + } + } +} + +// This assumes that the fields list has been obtained via BlockOutputField::iter().collect_vec(). +fn expected_header_for_default_field_order() -> &'static str { + "height,is_mainchain,id,timestamp,pool_id,target,chain_trust,status,parent_id" +} + +fn expected_genesis_output_line_for_default_field_order(chain_config: &ChainConfig) -> String { + let id = chain_config.genesis_block_id(); + let ts = chain_config.genesis_block().timestamp().as_int_seconds(); + let status = BlockStatusOutput::Good.to_string(); + format!("0,y,{id:x},{ts},-,-,-,{status},-") +} + +// Same assumption about field list order as above. +fn expected_output_line_for_default_field_order( + info: &TestBlockInfo, + chain_config: &ChainConfig, +) -> String { + let height = info.input_info.height; + let is_mc = if info.input_info.is_mainchain { + "y" + } else { + "n" + }; + let id = info.block_index.block_id(); + let ts = info.input_info.timestamp.as_int_seconds(); + let pool_id = Address::new(chain_config, info.input_info.pool_id).unwrap().into_string(); + let target = info.input_info.target; + let ctrust = info.input_info.chain_trust; + let status = info.input_info.status.to_string(); + let parent_id = info.input_info.parent_id; + + format!("{height},{is_mc},{id:x},{ts},{pool_id},{target:x},{ctrust:x},{status},{parent_id:x}") +} + +fn bad_block_status() -> BlockStatus { + let mut status = BlockStatus::new(); + status.set_validation_failed(); + status +} + +fn make_consensus_data(pool_id: PoolId, compact_target: Compact) -> ConsensusData { + // Create a new rng based on pool id, so that this function can be used in deterministic scenarios. + let mut rng = make_seedable_rng(Seed(pool_id.as_hash().to_low_u64_le())); + let vrf_return = bogus_vrf_return(&mut rng); + + ConsensusData::PoS(Box::new(PoSData::new( + vec![], + vec![], + pool_id, + vrf_return, + compact_target, + ))) +} + +fn bogus_vrf_return(rng: &mut (impl Rng + CryptoRng)) -> VRFReturn { + let (vrf_sk, _) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + let vrf_transcript = construct_transcript( + rng.gen(), + &rng.gen(), + BlockTimestamp::from_int_seconds(rng.gen()), + ) + .with_rng(rng); + + vrf_sk.produce_vrf_data(vrf_transcript) +} + +fn gen_compact_target(rng: &mut (impl Rng + CryptoRng)) -> Compact { + let target = Uint256::from_bytes(rng.gen()); + target.into() +} + +fn gen_target(rng: &mut (impl Rng + CryptoRng)) -> Uint256 { + gen_compact_target(rng).try_into().unwrap() +} + +fn id_from_str(s: &str) -> Id { + Id::new(H256::from_str(s).unwrap()) +} + +fn pool_id_from_str(s: &str, chain_config: &ChainConfig) -> PoolId { + Address::from_string(chain_config, s.to_owned()).unwrap().into_object() +} + +fn uint256_from_str(s: &str) -> Uint256 { + let data = hex::decode(s).unwrap(); + Uint256::from_be_slice(&data).unwrap() +} diff --git a/chainstate/db-dumper/src/dumper_lib/utils.rs b/chainstate/db-dumper/src/dumper_lib/utils.rs new file mode 100644 index 0000000000..637ab6fd41 --- /dev/null +++ b/chainstate/db-dumper/src/dumper_lib/utils.rs @@ -0,0 +1,95 @@ +// Copyright (c) 2021-2025 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{path::PathBuf, sync::Arc}; + +use chainstate::{ + BlockIndex, ChainstateError, ChainstateSubsystem, DefaultTransactionVerificationStrategy, +}; +use chainstate_launcher::check_storage_compatibility; +use chainstate_storage::Transactional as _; +use common::chain::{ + block::{consensus_data::PoSData, ConsensusData}, + ChainConfig, +}; +use logging::log; +use storage_lmdb::resize_callback::MapResizeCallback; + +use crate::Error; + +pub fn write_eol(output: &mut impl std::io::Write) -> Result<(), Error> { + writeln!(output).map_err(map_output_write_err) +} + +pub fn write_comma_if_needed( + field_idx: usize, + fields: &[T], + output: &mut impl std::io::Write, +) -> Result<(), Error> { + if field_idx + 1 != fields.len() { + write!(output, ",").map_err(map_output_write_err)?; + } + + Ok(()) +} + +pub fn map_output_write_err(err: std::io::Error) -> Error { + Error::OutputWriteError(err.to_string()) +} + +pub fn create_chainstate( + chain_config: Arc, + db_path: PathBuf, +) -> Result { + let lmdb_resize_callback = MapResizeCallback::new(Box::new(|resize_info| { + log::warn!("Lmdb resize happened: {:?}", resize_info) + })); + + let storage_backend = storage_lmdb::Lmdb::new( + db_path, + Default::default(), + Default::default(), + lmdb_resize_callback, + ) + .make_read_only(); + + let storage = chainstate_storage::Store::from_backend(storage_backend) + .map_err(|e| ChainstateError::FailedToInitializeChainstate(e.into()))?; + + { + let db_tx = storage.transaction_ro().map_err(Error::StorageCreationError)?; + check_storage_compatibility(&db_tx, chain_config.as_ref())?; + } + + let chainstate = chainstate::make_chainstate( + chain_config, + Default::default(), + storage, + DefaultTransactionVerificationStrategy::new(), + None, + Default::default(), + )?; + + Ok(chainstate) +} + +pub fn get_pos_consensus_data(block_index: &BlockIndex) -> Result<&PoSData, Error> { + match block_index.block_header().consensus_data() { + ConsensusData::PoS(data) => Ok(data), + ConsensusData::None | ConsensusData::PoW(_) => { + Err(Error::NonPoSConsensusInBlock(*block_index.block_id())) + } + } +} diff --git a/chainstate/launcher/src/lib.rs b/chainstate/launcher/src/lib.rs index 99ac5eebf8..929c2fcce2 100644 --- a/chainstate/launcher/src/lib.rs +++ b/chainstate/launcher/src/lib.rs @@ -35,6 +35,8 @@ pub use config::{ChainstateLauncherConfig, StorageBackendConfig}; /// Subdirectory under `datadir` where LMDB chainstate database is placed pub const SUBDIRECTORY_LMDB: &str = "chainstate-lmdb"; +pub use storage_compatibility::check_storage_compatibility; + fn make_chainstate_and_storage_impl( storage_backend: B, chain_config: Arc, @@ -47,7 +49,7 @@ fn make_chainstate_and_storage_impl( .transaction_ro() .map_err(|e| Error::FailedToInitializeChainstate(e.into()))?; - storage_compatibility::check_storage_compatibility(&db_tx, chain_config.as_ref()) + check_storage_compatibility(&db_tx, chain_config.as_ref()) .map_err(InitializationError::StorageCompatibilityCheckError)?; drop(db_tx); diff --git a/chainstate/src/detail/bootstrap.rs b/chainstate/src/detail/bootstrap.rs index 9d9e56521b..359b2a4513 100644 --- a/chainstate/src/detail/bootstrap.rs +++ b/chainstate/src/detail/bootstrap.rs @@ -109,12 +109,12 @@ fn fill_buffer( pub fn export_bootstrap_stream<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy>( magic_bytes: &[u8], writer: &mut std::io::BufWriter>, - include_orphans: bool, + include_stale_blocks: bool, query_interface: &ChainstateQuery<'a, S, V>, ) -> Result<(), BootstrapError> where { - let blocks_list = if include_orphans { + let blocks_list = if include_stale_blocks { query_interface.get_block_id_tree_as_list()? } else { query_interface.get_mainchain_blocks_list()? diff --git a/chainstate/src/interface/chainstate_interface.rs b/chainstate/src/interface/chainstate_interface.rs index ed6799f7cb..d2ca4869e7 100644 --- a/chainstate/src/interface/chainstate_interface.rs +++ b/chainstate/src/interface/chainstate_interface.rs @@ -242,7 +242,8 @@ pub trait ChainstateInterface: Send + Sync { /// Returns a list of all block ids in mainchain in order (starting from block of height 1, hence the result length is best_height - 1). fn get_mainchain_blocks_list(&self) -> Result>, ChainstateError>; - /// Returns a list of all blocks in the block tree, including orphans. The length cannot be predicted before the call. + /// Returns a list of all blocks in the block tree, including stale blocks, ordered by block height. + /// The length cannot be predicted before the call. fn get_block_id_tree_as_list(&self) -> Result>, ChainstateError>; /// Imports a bootstrap file exported with `export_bootstrap_stream`. @@ -253,12 +254,10 @@ pub trait ChainstateInterface: Send + Sync { /// Writes the blocks of the blockchain into a stream that's meant to go to a file. /// The blocks in the stream can be used to resync the blockchain in another node. - /// NOTE: `include_orphans` here means "include all blocks that are not on mainchain", rather than just - /// "blocks without a parent". fn export_bootstrap_stream<'a>( &self, writer: std::io::BufWriter>, - include_orphans: bool, + include_stale_blocks: bool, ) -> Result<(), ChainstateError>; /// Returns the UTXO for a specified OutPoint. diff --git a/chainstate/src/interface/chainstate_interface_impl.rs b/chainstate/src/interface/chainstate_interface_impl.rs index 95998ad41f..411cdd8001 100644 --- a/chainstate/src/interface/chainstate_interface_impl.rs +++ b/chainstate/src/interface/chainstate_interface_impl.rs @@ -625,14 +625,14 @@ where fn export_bootstrap_stream<'a>( &self, writer: std::io::BufWriter>, - include_orphans: bool, + include_stale_blocks: bool, ) -> Result<(), ChainstateError> { let magic_bytes = self.chainstate.chain_config().magic_bytes(); let mut writer = writer; export_bootstrap_stream( &magic_bytes.bytes(), &mut writer, - include_orphans, + include_stale_blocks, &self.chainstate.query().map_err(ChainstateError::from)?, )?; Ok(()) diff --git a/chainstate/src/interface/chainstate_interface_impl_delegation.rs b/chainstate/src/interface/chainstate_interface_impl_delegation.rs index b5742ec68c..7f12286ba2 100644 --- a/chainstate/src/interface/chainstate_interface_impl_delegation.rs +++ b/chainstate/src/interface/chainstate_interface_impl_delegation.rs @@ -315,9 +315,9 @@ where fn export_bootstrap_stream<'a>( &self, writer: std::io::BufWriter>, - include_orphans: bool, + include_stale_blocks: bool, ) -> Result<(), ChainstateError> { - self.deref().export_bootstrap_stream(writer, include_orphans) + self.deref().export_bootstrap_stream(writer, include_stale_blocks) } fn utxo(&self, outpoint: &UtxoOutPoint) -> Result, ChainstateError> { diff --git a/chainstate/src/rpc/mod.rs b/chainstate/src/rpc/mod.rs index 91cc1f60db..85f8be8332 100644 --- a/chainstate/src/rpc/mod.rs +++ b/chainstate/src/rpc/mod.rs @@ -175,7 +175,7 @@ trait ChainstateRpc { async fn export_bootstrap_file( &self, file_path: &std::path::Path, - include_orphans: bool, + include_stale_blocks: bool, ) -> RpcResult<()>; /// Imports a bootstrap file's blocks to this node @@ -428,7 +428,7 @@ impl ChainstateRpcServer for super::ChainstateHandle { async fn export_bootstrap_file( &self, file_path: &std::path::Path, - include_orphans: bool, + include_stale_blocks: bool, ) -> RpcResult<()> { // TODO: test this function in functional tests let file_obj: std::fs::File = rpc::handle_result(std::fs::File::create(file_path))?; @@ -436,7 +436,7 @@ impl ChainstateRpcServer for super::ChainstateHandle { std::io::BufWriter::new(Box::new(file_obj)); rpc::handle_result( - self.call(move |this| this.export_bootstrap_stream(writer, include_orphans)) + self.call(move |this| this.export_bootstrap_stream(writer, include_stale_blocks)) .await, ) } diff --git a/mocks/src/chainstate.rs b/mocks/src/chainstate.rs index 9a8a9de594..cc02020f03 100644 --- a/mocks/src/chainstate.rs +++ b/mocks/src/chainstate.rs @@ -165,7 +165,7 @@ mockall::mock! { fn export_bootstrap_stream<'a>( &'a self, writer: std::io::BufWriter>, - include_orphans: bool, + include_stale_blocks: bool, ) -> Result<(), ChainstateError>; fn utxo(&self, outpoint: &UtxoOutPoint) -> Result, ChainstateError>; fn is_initial_block_download(&self) -> bool; diff --git a/node-daemon/docs/RPC.md b/node-daemon/docs/RPC.md index f5a1f77550..130c56f364 100644 --- a/node-daemon/docs/RPC.md +++ b/node-daemon/docs/RPC.md @@ -554,7 +554,7 @@ Parameters: ``` { "file_path": string, - "include_orphans": bool, + "include_stale_blocks": bool, } ``` diff --git a/node-lib/src/options.rs b/node-lib/src/options.rs index e946704a72..cb2a132ba0 100644 --- a/node-lib/src/options.rs +++ b/node-lib/src/options.rs @@ -33,7 +33,7 @@ use common::chain::{ }, }; use utils::{ - clap_utils, default_data_dir::default_data_dir_common, root_user::ForceRunAsRootOptions, + clap_utils, default_data_dir::default_data_dir_for_chain, root_user::ForceRunAsRootOptions, }; use utils_networking::IpOrSocketAddress; @@ -370,7 +370,7 @@ pub struct RunOptions { } pub fn default_data_dir(chain_type: ChainType) -> PathBuf { - default_data_dir_common().join(chain_type.name()) + default_data_dir_for_chain(chain_type.name()) } #[cfg(test)] diff --git a/storage/lmdb/src/lib.rs b/storage/lmdb/src/lib.rs index 2124c35c1d..27ab57493d 100644 --- a/storage/lmdb/src/lib.rs +++ b/storage/lmdb/src/lib.rs @@ -269,10 +269,30 @@ impl Lmdb { self } - fn open_db(env: &lmdb::Environment, desc: &DbMapDesc) -> storage_core::Result { + /// Open the db only for reading. + pub fn make_read_only(mut self) -> Self { + self.flags |= lmdb::EnvironmentFlags::READ_ONLY; + self + } + + fn is_read_only(&self) -> bool { + !(self.flags & lmdb::EnvironmentFlags::READ_ONLY).is_empty() + } + + fn open_or_create_db( + env: &lmdb::Environment, + desc: &DbMapDesc, + open_only: bool, + ) -> storage_core::Result { let name = Some(desc.name()); - let flags = lmdb::DatabaseFlags::default(); - env.create_db(name, flags).or_else(error::process_with_err) + + if open_only { + env.open_db(name) + } else { + let flags = lmdb::DatabaseFlags::default(); + env.create_db(name, flags) + } + .or_else(error::process_with_err) } } @@ -280,8 +300,11 @@ impl backend::Backend for Lmdb { type Impl = LmdbImpl; fn open(self, desc: DbDesc) -> storage_core::Result { - // Attempt to create the storage directory - std::fs::create_dir_all(&self.path).map_err(error::process_io_error)?; + let read_only = self.is_read_only(); + if !read_only { + // Attempt to create the storage directory + std::fs::create_dir_all(&self.path).map_err(error::process_io_error)?; + } let initial_map_size = self .initial_map_size @@ -304,7 +327,9 @@ impl backend::Backend for Lmdb { .or_else(error::process_with_err)?; // Set up all the databases - let dbs = desc.db_maps().try_transform(|desc| Self::open_db(&environment, desc))?; + let dbs = desc + .db_maps() + .try_transform(|desc| Self::open_or_create_db(&environment, desc, read_only))?; let dbs = dbs.into(); Ok(LmdbImpl {