diff --git a/.gitignore b/.gitignore index 8196c671..324cf2b3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ test-ledger/ minio test.db docker-compose.yml + +photon.log \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index ae825d0c..a7cfd5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,8 +105,7 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned-sized" version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48a526ec4434d531d488af59fe866f36b310fe8906691c75dffa664450a3800a" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "proc-macro2", "quote", @@ -3681,10 +3680,10 @@ dependencies = [ [[package]] name = "light-account-checks" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3fd000a2b8e0cc9d0b7b7712964870df51f2114f1693b9d8f0414f6f3ec16bd" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "solana-account-info", + "solana-msg", "solana-program-error", "solana-pubkey", "solana-sysvar", @@ -3694,15 +3693,14 @@ dependencies = [ [[package]] name = "light-batched-merkle-tree" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c7e179246468b09bf5c6882ef33043e178ff90eb6eab0c1c4c3623ef84b154" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "aligned-sized", "borsh 0.10.4", "light-account-checks", "light-bloom-filter", "light-compressed-account", - "light-hasher", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db)", "light-macros", "light-merkle-tree-metadata", "light-verifier", @@ -3719,8 +3717,7 @@ dependencies = [ [[package]] name = "light-bloom-filter" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44abcb5554e1c15cefa9ac17e4ceda6f5afb039db25ab1fd777f012356d0f964" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "bitvec", "num-bigint 0.4.6", @@ -3744,15 +3741,17 @@ dependencies = [ [[package]] name = "light-compressed-account" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f15113babaca9efb592631ec1e7e78c1c83413818a6e1e4248b7df53d88fe65" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "anchor-lang 0.31.1", "borsh 0.10.4", "bytemuck", - "light-hasher", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db)", "light-macros", + "light-profiler", "light-zero-copy", + "log", + "solana-msg", "solana-program-error", "solana-pubkey", "thiserror 2.0.12", @@ -3767,7 +3766,7 @@ checksum = "9b4f878301620df78ba7e7758c5fd720f28040f5c157375f88d310f15ddb1746" dependencies = [ "borsh 0.10.4", "light-bounded-vec", - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.9.1", "thiserror 2.0.12", ] @@ -3777,6 +3776,23 @@ name = "light-hasher" version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" +dependencies = [ + "ark-bn254 0.5.0", + "ark-ff 0.5.0", + "arrayvec", + "borsh 0.10.4", + "light-poseidon 0.3.0", + "num-bigint 0.4.6", + "sha2 0.10.9", + "sha3 0.10.8", + "solana-nostd-keccak", + "thiserror 2.0.12", +] + +[[package]] +name = "light-hasher" +version = "3.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3798,7 +3814,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc786d8df68ef64493fea04914a7a7745f8122f2efbae043cd4ba4eaffa9e6db" dependencies = [ - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.4.6", "num-traits", "thiserror 2.0.12", @@ -3807,8 +3823,7 @@ dependencies = [ [[package]] name = "light-macros" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "861c0817697c1201c2235cd831fcbaa2564a5f778e5229e9f5cc21035e97c273" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "bs58 0.5.1", "proc-macro2", @@ -3819,8 +3834,7 @@ dependencies = [ [[package]] name = "light-merkle-tree-metadata" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "544048fa95ea95fc1e952a2b9b1d6f09340c8decaffd1ad239fe1f6eb905ae76" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "borsh 0.10.4", "bytemuck", @@ -3838,7 +3852,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1650701feac958261b2c3ab4da361ad8548985ee3ee496a17e76db44d2d3c9e3" dependencies = [ - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "light-indexed-array", "num-bigint 0.4.6", "num-traits", @@ -3869,11 +3883,20 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "light-profiler" +version = "0.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "light-verifier" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fdf317ec3cfcd3a8e6556a5b5e7fbcc207a40264700f9a5271876838f26f58" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "groth16-solana", "light-compressed-account", @@ -3883,14 +3906,25 @@ dependencies = [ [[package]] name = "light-zero-copy" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34d759f65547a6540db7047f38f4cb2c3f01658deca95a1dd06f26b578de947" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ + "arrayvec", + "light-zero-copy-derive", "solana-program-error", - "thiserror 2.0.12", "zerocopy", ] +[[package]] +name = "light-zero-copy-derive" +version = "0.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" +dependencies = [ + "lazy_static", + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -4533,7 +4567,7 @@ dependencies = [ [[package]] name = "photon-indexer" -version = "0.51.0" +version = "0.52.3" dependencies = [ "anchor-lang 0.29.0", "anyhow", @@ -4564,7 +4598,7 @@ dependencies = [ "light-batched-merkle-tree", "light-compressed-account", "light-concurrent-merkle-tree", - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "light-merkle-tree-metadata", "light-merkle-tree-reference", "light-poseidon 0.3.0", diff --git a/Cargo.toml b/Cargo.toml index b0e81c70..60a01ff8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.51.0" +version = "0.52.4" [[bin]] name = "photon" @@ -82,11 +82,21 @@ solana-pubkey = "2.3.0" solana-transaction-status = "1.18.0" light-concurrent-merkle-tree = "2.1.0" -light-batched-merkle-tree = "0.3.0" -light-merkle-tree-metadata = "0.3.0" -light-compressed-account = { version = "0.3.0", features = ["anchor"] } -light-hasher = { version = "3.1.0" } +# light-batched-merkle-tree = "0.3.0" +# light-merkle-tree-metadata = "0.3.0" +# light-compressed-account = { version = "0.3.0", features = ["anchor"] } + +light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } +light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } +light-compressed-account = { version = "0.3.0", features = [ "anchor"], git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } + +# light-batched-merkle-tree = { path = "../light-protocol/program-libs/batched-merkle-tree" } +# light-merkle-tree-metadata = { path = "../light-protocol/program-libs/merkle-tree-metadata" } +# light-compressed-account = { features = ["anchor"], path = "../light-protocol/program-libs/compressed-account" } + + +light-hasher = { version = "3.1.0" } light-poseidon = "0.3.0" sqlx = { version = "0.6.2", features = [ diff --git a/src/api/method/get_compressed_account_balance.rs b/src/api/method/get_compressed_account_balance.rs index 418d5c4b..9909ca32 100644 --- a/src/api/method/get_compressed_account_balance.rs +++ b/src/api/method/get_compressed_account_balance.rs @@ -5,7 +5,6 @@ use crate::common::typedefs::context::Context; use crate::common::typedefs::unsigned_integer::UnsignedInteger; use crate::dao::generated::accounts; use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect}; -use sqlx::types::Decimal; pub async fn get_compressed_account_balance( conn: &DatabaseConnection, @@ -22,7 +21,7 @@ pub async fn get_compressed_account_balance( .one(conn) .await? .map(|x| x.lamports) - .unwrap_or(Decimal::from(0)); + .unwrap_or_else(|| "0".to_string()); Ok(AccountBalanceResponse { value: UnsignedInteger(parse_decimal(balance)?), diff --git a/src/api/method/get_compressed_balance_by_owner.rs b/src/api/method/get_compressed_balance_by_owner.rs index e5ebfebd..5e0b06f1 100644 --- a/src/api/method/get_compressed_balance_by_owner.rs +++ b/src/api/method/get_compressed_balance_by_owner.rs @@ -28,7 +28,7 @@ pub async fn get_compressed_balance_by_owner( .into_model::() .all(conn) .await? - .iter() + .into_iter() .map(|x| parse_decimal(x.lamports)) .collect::, PhotonApiError>>()?; diff --git a/src/api/method/get_compressed_mint_token_holders.rs b/src/api/method/get_compressed_mint_token_holders.rs index c2ae1e98..ddea4edc 100644 --- a/src/api/method/get_compressed_mint_token_holders.rs +++ b/src/api/method/get_compressed_mint_token_holders.rs @@ -7,7 +7,7 @@ use crate::common::typedefs::bs58_string::Base58String; use crate::common::typedefs::context::Context; use crate::common::typedefs::limit::Limit; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_owner_balances; use super::super::error::PhotonApiError; @@ -16,6 +16,7 @@ use super::utils::{parse_decimal, PAGE_LIMIT}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct OwnerBalance { pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub balance: UnsignedInteger, } diff --git a/src/api/method/get_compressed_token_account_balance.rs b/src/api/method/get_compressed_token_account_balance.rs index a6fbbdd0..95fecd03 100644 --- a/src/api/method/get_compressed_token_account_balance.rs +++ b/src/api/method/get_compressed_token_account_balance.rs @@ -1,4 +1,4 @@ -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_accounts; use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect}; use serde::{Deserialize, Serialize}; @@ -7,7 +7,6 @@ use super::super::error::PhotonApiError; use super::utils::{parse_decimal, AccountDataTable}; use super::utils::{BalanceModel, CompressedAccountRequest}; use crate::common::typedefs::context::Context; -use sqlx::types::Decimal; use utoipa::ToSchema; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] @@ -15,6 +14,7 @@ use utoipa::ToSchema; // This is a struct because in the future we might add other fields here like decimals or uiAmount, // which is a string representation with decimals in the form of "10.00" pub struct TokenAccountBalance { + #[serde(serialize_with = "serialize_u64_as_string")] pub amount: UnsignedInteger, } @@ -40,7 +40,7 @@ pub async fn get_compressed_token_account_balance( .one(conn) .await? .map(|x| x.amount) - .unwrap_or(Decimal::from(0)); + .unwrap_or_else(|| "0".to_string()); Ok(GetCompressedTokenAccountBalanceResponse { value: TokenAccountBalance { diff --git a/src/api/method/get_compressed_token_balances_by_owner.rs b/src/api/method/get_compressed_token_balances_by_owner.rs index 4f888fd4..7e5cbf14 100644 --- a/src/api/method/get_compressed_token_balances_by_owner.rs +++ b/src/api/method/get_compressed_token_balances_by_owner.rs @@ -6,7 +6,7 @@ use crate::common::typedefs::bs58_string::Base58String; use crate::common::typedefs::context::Context; use crate::common::typedefs::limit::Limit; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_owner_balances; use super::super::error::PhotonApiError; @@ -15,6 +15,7 @@ use super::utils::{parse_decimal, PAGE_LIMIT}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct TokenBalance { pub mint: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub balance: UnsignedInteger, } diff --git a/src/api/method/utils.rs b/src/api/method/utils.rs index 6e532504..8948af3c 100644 --- a/src/api/method/utils.rs +++ b/src/api/method/utils.rs @@ -4,7 +4,7 @@ use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::serializable_signature::SerializableSignature; use crate::common::typedefs::token_data::{AccountState, TokenData}; use crate::common::typedefs::unix_timestamp::UnixTimestamp; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::{accounts, token_accounts}; use byteorder::{ByteOrder, LittleEndian}; @@ -28,11 +28,18 @@ use super::super::error::PhotonApiError; pub const PAGE_LIMIT: u64 = 1000; -pub fn parse_decimal(value: Decimal) -> Result { - value - .to_string() +// Avoids precision loss +pub fn parse_decimal(value: String) -> Result { + Ok(value .parse::() - .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal value".to_string())) + .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal string".to_string()))?) +} + +// Avoids precision loss +pub fn parse_u64_string(value: String) -> Result { + Ok(value + .parse::() + .map_err(|_| PhotonApiError::UnexpectedError("Invalid discriminator string".to_string()))?) } pub(crate) fn parse_leaf_index(leaf_index: i64) -> Result { @@ -307,12 +314,12 @@ impl CompressedAccountRequest { #[derive(FromQueryResult)] pub struct BalanceModel { - pub amount: Decimal, + pub amount: String, } #[derive(FromQueryResult)] pub struct LamportModel { - pub lamports: Decimal, + pub lamports: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema, Default)] @@ -608,6 +615,7 @@ pub struct GetPaginatedSignaturesResponse { // We do not use generics to simplify documentation generation. pub struct AccountBalanceResponse { pub context: Context, + #[serde(serialize_with = "serialize_u64_as_string")] pub value: UnsignedInteger, } diff --git a/src/common/typedefs/account/context.rs b/src/common/typedefs/account/context.rs index ac795925..46c08fbc 100644 --- a/src/common/typedefs/account/context.rs +++ b/src/common/typedefs/account/context.rs @@ -1,5 +1,5 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::{parse_decimal, parse_leaf_index}; +use crate::api::method::utils::{parse_decimal, parse_leaf_index, parse_u64_string}; use crate::common::typedefs::account::{Account, AccountData}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; @@ -71,10 +71,13 @@ impl AccountWithContext { data, } = compressed_account; - let data = data.map(|d| AccountData { - discriminator: UnsignedInteger(LittleEndian::read_u64(&d.discriminator)), - data: Base64String(d.data), - data_hash: Hash::from(d.data_hash), + let data = data.map(|d| { + let discriminator_u64 = LittleEndian::read_u64(&d.discriminator); + AccountData { + discriminator: UnsignedInteger(discriminator_u64), + data: Base64String(d.data), + data_hash: Hash::from(d.data_hash), + } }); Self { @@ -111,7 +114,7 @@ impl TryFrom for AccountWithContext { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index 05d83051..52a13c1e 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -1,9 +1,9 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::parse_decimal; +use crate::api::method::utils::{parse_decimal, parse_u64_string}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::accounts::Model; use jsonrpsee_core::Serialize; use utoipa::ToSchema; @@ -15,6 +15,7 @@ pub struct Account { pub address: Option, pub data: Option, pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub lamports: UnsignedInteger, pub tree: SerializablePubkey, pub leaf_index: UnsignedInteger, @@ -29,6 +30,7 @@ pub struct Account { #[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema, Default)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AccountData { + #[serde(serialize_with = "serialize_u64_as_string")] pub discriminator: UnsignedInteger, pub data: Base64String, pub data_hash: Hash, @@ -42,7 +44,7 @@ impl TryFrom for Account { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { @@ -70,3 +72,138 @@ impl TryFrom for Account { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_discriminator_serializes_as_string() { + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account_data = AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1, 2, 3]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "Discriminator should be serialized as string, got: {}", + json + ); + + + assert!( + !json.contains(&format!("\"discriminator\":{}", expected_u64)), + "Discriminator should not be serialized as number, got: {}", + json + ); + } + + #[test] + fn test_discriminator_prevents_javascript_precision_loss() { + let large_discriminator = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 + + let account_data = AccountData { + discriminator: UnsignedInteger(large_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", large_discriminator)), + "Large discriminator should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } + + #[test] + fn test_discriminator_with_max_u64() { + let max_discriminator = u64::MAX; + + let account_data = AccountData { + discriminator: UnsignedInteger(max_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), + "MAX u64 discriminator should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_discriminator_zero_value() { + let account_data = AccountData { + discriminator: UnsignedInteger(0), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains("\"discriminator\":\"0\""), + "Zero discriminator should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_lamports_serializes_as_string() { + let account = Account { + hash: Hash::default(), + address: None, + data: None, + owner: SerializablePubkey::default(), + lamports: UnsignedInteger(1000000000), + tree: SerializablePubkey::default(), + leaf_index: UnsignedInteger(0), + seq: Some(UnsignedInteger(1)), + slot_created: UnsignedInteger(100), + }; + + let json = serde_json::to_string(&account).unwrap(); + + assert!( + json.contains("\"lamports\":\"1000000000\""), + "Lamports should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_lamports_prevents_javascript_precision_loss() { + let large_lamports = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 + + let account = Account { + hash: Hash::default(), + address: None, + data: None, + owner: SerializablePubkey::default(), + lamports: UnsignedInteger(large_lamports), + tree: SerializablePubkey::default(), + leaf_index: UnsignedInteger(0), + seq: None, + slot_created: UnsignedInteger(0), + }; + + let json = serde_json::to_string(&account).unwrap(); + + assert!( + json.contains(&format!("\"lamports\":\"{}\"", large_lamports)), + "Large lamports should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } +} diff --git a/src/common/typedefs/account/v2.rs b/src/common/typedefs/account/v2.rs index 996d23b0..2e1a6aa8 100644 --- a/src/common/typedefs/account/v2.rs +++ b/src/common/typedefs/account/v2.rs @@ -1,11 +1,11 @@ use crate::api::error::PhotonApiError; use crate::api::method::get_validity_proof::MerkleContextV2; -use crate::api::method::utils::parse_decimal; +use crate::api::method::utils::{parse_decimal, parse_u64_string}; use crate::common::typedefs::account::AccountData; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::accounts::Model; use serde::Serialize; use utoipa::ToSchema; @@ -17,6 +17,7 @@ pub struct AccountV2 { pub address: Option, pub data: Option, pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub lamports: UnsignedInteger, pub leaf_index: UnsignedInteger, // For legacy trees is always Some() since the user tx appends directly to the Merkle tree @@ -42,7 +43,7 @@ impl TryFrom for AccountV2 { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/token_data.rs b/src/common/typedefs/token_data.rs index e7ad01b3..402a8136 100644 --- a/src/common/typedefs/token_data.rs +++ b/src/common/typedefs/token_data.rs @@ -5,7 +5,7 @@ use utoipa::ToSchema; use super::{ bs64_string::Base64String, serializable_pubkey::SerializablePubkey, - unsigned_integer::UnsignedInteger, + unsigned_integer::{serialize_u64_as_string, UnsignedInteger}, }; #[derive( @@ -40,6 +40,7 @@ pub struct TokenData { /// The owner of this account. pub owner: SerializablePubkey, /// The amount of tokens this account holds. + #[serde(serialize_with = "serialize_u64_as_string")] pub amount: UnsignedInteger, /// If `delegate` is `Some` then `delegated_amount` represents /// the amount authorized by the delegate @@ -49,3 +50,70 @@ pub struct TokenData { /// Placeholder for TokenExtension tlv data (unimplemented) pub tlv: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_amount_serializes_as_string() { + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(1000000000), + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + assert!( + json.contains("\"amount\":\"1000000000\""), + "Amount should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_token_amount_prevents_javascript_precision_loss() { + let large_amount = u64::MAX; + + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(large_amount), + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + assert!( + json.contains(&format!("\"amount\":\"{}\"", large_amount)), + "Large amount should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } + + #[test] + fn test_token_amount_zero_value() { + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(0), + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + assert!( + json.contains("\"amount\":\"0\""), + "Zero amount should be serialized as string, got: {}", + json + ); + } +} diff --git a/src/common/typedefs/unsigned_integer.rs b/src/common/typedefs/unsigned_integer.rs index 542fcd53..9b2c8ec0 100644 --- a/src/common/typedefs/unsigned_integer.rs +++ b/src/common/typedefs/unsigned_integer.rs @@ -78,7 +78,7 @@ impl anchor_lang::AnchorDeserialize for UnsignedInteger { } fn deserialize_reader(reader: &mut R) -> Result { - let mut buffer = [0u8; 8]; // Adjusting the size for u64 + let mut buffer = [0u8; 8]; reader.read_exact(&mut buffer)?; let value = u64::from_le_bytes(buffer); Ok(UnsignedInteger(value)) @@ -90,3 +90,15 @@ impl anchor_lang::AnchorSerialize for UnsignedInteger { writer.write_all(&self.0.to_le_bytes()) } } + +/// Serialize u64 as string to prevent precision loss if using SQLite. +pub fn serialize_u64_as_string( + u64_value: &UnsignedInteger, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let u64_string = u64_value.0.to_string(); + serializer.serialize_str(&u64_string) +} diff --git a/src/dao/generated/accounts.rs b/src/dao/generated/accounts.rs index c968a218..04cb19b0 100644 --- a/src/dao/generated/accounts.rs +++ b/src/dao/generated/accounts.rs @@ -17,10 +17,8 @@ pub struct Model { pub slot_created: i64, pub spent: bool, pub prev_spent: Option, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub lamports: Decimal, - #[sea_orm(column_type = "Decimal(Some((23, 0)))", nullable)] - pub discriminator: Option, + pub lamports: String, + pub discriminator: Option, pub tree_type: Option, pub nullified_in_tree: bool, pub nullifier_queue_index: Option, diff --git a/src/dao/generated/owner_balances.rs b/src/dao/generated/owner_balances.rs index 2d4b20fb..ca359202 100644 --- a/src/dao/generated/owner_balances.rs +++ b/src/dao/generated/owner_balances.rs @@ -7,8 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub owner: Vec, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub lamports: Decimal, + pub lamports: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/dao/generated/token_accounts.rs b/src/dao/generated/token_accounts.rs index e9604cba..25df6d04 100644 --- a/src/dao/generated/token_accounts.rs +++ b/src/dao/generated/token_accounts.rs @@ -13,8 +13,7 @@ pub struct Model { pub state: i32, pub spent: bool, pub prev_spent: Option, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub amount: Decimal, + pub amount: String, pub tlv: Option>, } diff --git a/src/dao/generated/token_owner_balances.rs b/src/dao/generated/token_owner_balances.rs index 499c8518..f88f411b 100644 --- a/src/dao/generated/token_owner_balances.rs +++ b/src/dao/generated/token_owner_balances.rs @@ -9,8 +9,7 @@ pub struct Model { pub owner: Vec, #[sea_orm(primary_key, auto_increment = false)] pub mint: Vec, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub amount: Decimal, + pub amount: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/ingester/parser/mod.rs b/src/ingester/parser/mod.rs index aedff22f..9f482707 100644 --- a/src/ingester/parser/mod.rs +++ b/src/ingester/parser/mod.rs @@ -65,6 +65,7 @@ pub fn parse_transaction(tx: &TransactionInfo, slot: u64) -> Result 1 { if get_compression_program_id() == instruction.program_id { diff --git a/src/ingester/parser/tree_info.rs b/src/ingester/parser/tree_info.rs index 3c073a7d..136b01a9 100644 --- a/src/ingester/parser/tree_info.rs +++ b/src/ingester/parser/tree_info.rs @@ -259,6 +259,10 @@ lazy_static! { pubkey!("bmt5yU97jC88YXTuSukYHa8Z5Bi2ZDUtmzfkDTA2mG2"), pubkey!("oq5oh5ZR3yGomuQgFduNDzjtGvVWfDRGLuDVjv9a96P"), ), + ( + pubkey!("2Yb3fGo2E9aWLjY8KuESaqurYpGGhEeJr7eynKrSgXwS"), + pubkey!("12wJT3xYd46rtjeqDU6CrtT8unqLjPiheggzqhN9YsyB"), + ), ]; let address_trees_v2 = [ diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index d8f6a704..26d9cbea 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -245,12 +245,34 @@ async fn persist_state_tree_history( pub fn parse_token_data(account: &Account) -> Result, IngesterError> { match account.data.clone() { - Some(data) if account.owner.0 == COMPRESSED_TOKEN_PROGRAM => { - let data_slice = data.data.0.as_slice(); - let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { - IngesterError::ParserError(format!("Failed to parse token data: {:?}", e)) - })?; - Ok(Some(token_data)) + Some(data) => { + let is_v1_token = data.discriminator.0.to_le_bytes() == [2, 0, 0, 0, 0, 0, 0, 0]; // V1 discriminator + let is_v2_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 3]; // V2 discriminator + let is_sha_flat_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 4]; // V3 discriminator + + // TODO: remove after debugging. + // println!("ingested data.discriminator: {:?}", data.discriminator); + // println!("is_v1_token: {:?}", is_v1_token); + // println!("is_v2_token: {:?}", is_v2_token); + // println!("is_sha_flat_token: {:?}", is_sha_flat_token); + // println!("account.owner.0: {:?}", account.owner.0); + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token) {} + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token || is_sha_flat_token) { + let data_slice = data.data.0.as_slice(); + let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { + IngesterError::ParserError(format!("Failed to parse token data: {:?}", e)) + })?; + Ok(Some(token_data)) + } else { + // TODO: remove after debugging. + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM { + // println!("Must be mint account. address: {:?}", account.address); + } + else { + // println!("Not mint account. address: {:?}", account.address); + } + Ok(None) + } } _ => Ok(None), } @@ -328,14 +350,12 @@ async fn execute_account_update_query_and_update_balances( let prev_spent: Option = row.try_get("", "prev_spent")?; match (prev_spent, &modification_type) { (_, ModificationType::Append) | (Some(false), ModificationType::Spend) => { - let mut amount_of_interest = match db_backend { - DatabaseBackend::Postgres => row.try_get("", balance_column)?, - DatabaseBackend::Sqlite => { - let amount: i64 = row.try_get("", balance_column)?; - Decimal::from(amount) - } - _ => panic!("Unsupported database backend"), - }; + let amount_str: String = row.try_get("", balance_column)?; + let amount_u64 = amount_str.parse::().map_err(|_| { + IngesterError::DatabaseError(format!("Invalid amount string: {}", amount_str)) + })?; + let amount_i64 = amount_u64 as i64; + let mut amount_of_interest = Decimal::from(amount_i64); amount_of_interest *= multiplier; let owner = bytes_to_sql_format(db_backend, row.try_get("", "owner")?); let key = match account_type { @@ -359,16 +379,35 @@ async fn execute_account_update_query_and_update_balances( let values = balance_modifications .into_iter() .filter(|(_, value)| *value != Decimal::from(0)) - .map(|(key, value)| format!("({}, {})", key, value)) + .map(|(key, value)| { + if db_backend == DatabaseBackend::Sqlite { + // Store as text + let value_i64: i64 = value.try_into().unwrap_or(0); + let value_u64 = value_i64.unsigned_abs(); + format!("({}, '{}')", key, value_u64) + } else { + // PostgreSQL + format!("({}, {})", key, value) + } + }) .collect::>(); if !values.is_empty() { let values_string = values.join(", "); - let raw_sql = format!( - "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) - VALUES {values_string} ON CONFLICT (owner{additional_columns}) - DO UPDATE SET {balance_column} = {owner_table_name}.{balance_column} + excluded.{balance_column}", - ); + let raw_sql = if txn.get_database_backend() == DatabaseBackend::Sqlite { + // For SQLite with TEXT columns, we need to cast to INTEGER for arithmetic + format!( + "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) + VALUES {values_string} ON CONFLICT (owner{additional_columns}) + DO UPDATE SET {balance_column} = CAST(CAST({owner_table_name}.{balance_column} AS INTEGER) + CAST(excluded.{balance_column} AS INTEGER) AS TEXT)", + ) + } else { + format!( + "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) + VALUES {values_string} ON CONFLICT (owner{additional_columns}) + DO UPDATE SET {balance_column} = {owner_table_name}.{balance_column} + excluded.{balance_column}", + ) + }; txn.execute(Statement::from_string(db_backend, raw_sql)) .await?; } @@ -413,11 +452,10 @@ async fn append_output_accounts( account_models.push(accounts::ActiveModel { hash: Set(account.account.hash.to_vec()), address: Set(account.account.address.map(|x| x.to_bytes_vec())), - discriminator: Set(account - .account - .data - .as_ref() - .map(|x| Decimal::from(x.discriminator.0))), + discriminator: Set(account.account.data.as_ref().map(|x| { + let discriminator_string = x.discriminator.0.to_string(); + discriminator_string + })), data: Set(account.account.data.as_ref().map(|x| x.data.clone().0)), data_hash: Set(account.account.data.as_ref().map(|x| x.data_hash.to_vec())), tree: Set(account.account.tree.to_bytes_vec()), @@ -429,7 +467,7 @@ async fn append_output_accounts( tree_type: Set(Some(account.context.tree_type as i32)), nullifier: Set(account.context.nullifier.as_ref().map(|x| x.to_vec())), owner: Set(account.account.owner.to_bytes_vec()), - lamports: Set(Decimal::from(account.account.lamports.0)), + lamports: Set(account.account.lamports.0.to_string()), spent: Set(false), slot_created: Set(account.account.slot_created.0 as i64), seq: Set(account.account.seq.map(|x| x.0 as i64)), @@ -481,7 +519,7 @@ pub async fn persist_token_accounts( hash: Set(hash.into()), mint: Set(token_data.mint.to_bytes_vec()), owner: Set(token_data.owner.to_bytes_vec()), - amount: Set(Decimal::from(token_data.amount.0)), + amount: Set(token_data.amount.0.to_string()), delegate: Set(token_data.delegate.map(|d| d.to_bytes_vec())), state: Set(token_data.state as i32), spent: Set(false), diff --git a/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs b/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs new file mode 100644 index 00000000..733cd5bb --- /dev/null +++ b/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs @@ -0,0 +1,142 @@ +use sea_orm_migration::{ + prelude::*, + sea_orm::{ConnectionTrait, DatabaseBackend, Statement}, +}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +async fn execute_sql<'a>(manager: &SchemaManager<'_>, sql: &str) -> Result<(), DbErr> { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + sql.to_string(), + )) + .await?; + Ok(()) +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Only apply this migration to SQLite - PostgreSQL already uses correct bigint2 type + if manager.get_database_backend() == DatabaseBackend::Sqlite { + execute_sql( + manager, + r#" + -- Fix discriminator precision loss by changing from REAL to TEXT + -- Step 1: Create new table with TEXT discriminator column + CREATE TABLE accounts_discriminator_fix ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, + discriminator TEXT, -- Changed from REAL to TEXT + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + -- Step 2: Copy data, converting REAL discriminator to TEXT + -- Note: This will lose precision for existing corrupted data, + -- but new data will be stored correctly as TEXT + INSERT INTO accounts_discriminator_fix + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, lamports, + CASE + WHEN discriminator IS NOT NULL THEN CAST(discriminator AS TEXT) + ELSE NULL + END as discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + -- Step 3: Drop old table and rename new one + DROP TABLE accounts; + ALTER TABLE accounts_discriminator_fix RENAME TO accounts; + + -- Step 4: Recreate all indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + + + "#, + ) + .await?; + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Rollback: Convert TEXT back to REAL (will cause precision loss again) + if manager.get_database_backend() == DatabaseBackend::Sqlite { + execute_sql( + manager, + r#" + -- Rollback discriminator from TEXT to REAL + CREATE TABLE accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, + discriminator REAL, -- Back to REAL + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + INSERT INTO accounts_rollback + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, lamports, + CASE + WHEN discriminator IS NOT NULL THEN CAST(discriminator AS REAL) + ELSE NULL + END as discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + DROP TABLE accounts; + ALTER TABLE accounts_rollback RENAME TO accounts; + + -- Recreate indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + } + + Ok(()) + } +} diff --git a/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs b/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs new file mode 100644 index 00000000..db1f74da --- /dev/null +++ b/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs @@ -0,0 +1,318 @@ +use sea_orm_migration::{ + prelude::*, + sea_orm::{ConnectionTrait, DatabaseBackend, Statement}, +}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +async fn execute_sql<'a>(manager: &SchemaManager<'_>, sql: &str) -> Result<(), DbErr> { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + sql.to_string(), + )) + .await?; + Ok(()) +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Only apply this migration to SQLite - PostgreSQL already uses correct bigint2 type + if manager.get_database_backend() == DatabaseBackend::Sqlite { + // Fix lamports in accounts table + execute_sql( + manager, + r#" + -- Fix lamports precision loss by changing from REAL to TEXT + -- Step 1: Create new table with TEXT lamports column + CREATE TABLE accounts_lamports_fix ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports TEXT, -- Changed from REAL to TEXT + discriminator TEXT, + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + -- Step 2: Copy data, converting REAL lamports to TEXT + INSERT INTO accounts_lamports_fix + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, + CASE + WHEN lamports IS NOT NULL THEN CAST(CAST(lamports AS INTEGER) AS TEXT) + ELSE NULL + END as lamports, + discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + -- Step 3: Drop old table and rename new one + DROP TABLE accounts; + ALTER TABLE accounts_lamports_fix RENAME TO accounts; + + -- Step 4: Recreate all indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + + // Fix amount in token_accounts table + execute_sql( + manager, + r#" + -- Fix amount precision loss in token_accounts + CREATE TABLE token_accounts_amount_fix ( + hash BLOB NOT NULL PRIMARY KEY, + owner BLOB NOT NULL, + mint BLOB NOT NULL, + delegate BLOB, + state INTEGER NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + amount TEXT, -- Changed from REAL to TEXT + tlv BLOB, + FOREIGN KEY (hash) REFERENCES accounts(hash) ON DELETE CASCADE + ); + + INSERT INTO token_accounts_amount_fix + SELECT + hash, owner, mint, delegate, state, spent, prev_spent, + CASE + WHEN amount IS NOT NULL THEN CAST(CAST(amount AS INTEGER) AS TEXT) + ELSE NULL + END as amount, + tlv + FROM token_accounts; + + DROP TABLE token_accounts; + ALTER TABLE token_accounts_amount_fix RENAME TO token_accounts; + + -- Recreate indexes + CREATE UNIQUE INDEX token_accounts_owner_mint_hash_idx ON token_accounts (spent, owner, mint, hash); + CREATE UNIQUE INDEX token_accounts_delegate_mint_hash_idx ON token_accounts (spent, delegate, mint, hash); + "#, + ) + .await?; + + // Fix lamports in owner_balances table + execute_sql( + manager, + r#" + -- Fix lamports precision loss in owner_balances + CREATE TABLE owner_balances_lamports_fix ( + owner BLOB NOT NULL PRIMARY KEY, + lamports TEXT -- Changed from REAL to TEXT + ); + + INSERT INTO owner_balances_lamports_fix + SELECT + owner, + CASE + WHEN lamports IS NOT NULL THEN CAST(CAST(lamports AS INTEGER) AS TEXT) + ELSE '0' + END as lamports + FROM owner_balances; + + DROP TABLE owner_balances; + ALTER TABLE owner_balances_lamports_fix RENAME TO owner_balances; + "#, + ) + .await?; + + // Fix amount in token_owner_balances table + execute_sql( + manager, + r#" + -- Fix amount precision loss in token_owner_balances + CREATE TABLE token_owner_balances_amount_fix ( + owner BLOB NOT NULL, + mint BLOB NOT NULL, + amount TEXT, -- Changed from REAL to TEXT + PRIMARY KEY (owner, mint) + ); + + INSERT INTO token_owner_balances_amount_fix + SELECT + owner, mint, + CASE + WHEN amount IS NOT NULL THEN CAST(CAST(amount AS INTEGER) AS TEXT) + ELSE '0' + END as amount + FROM token_owner_balances; + + DROP TABLE token_owner_balances; + ALTER TABLE token_owner_balances_amount_fix RENAME TO token_owner_balances; + "#, + ) + .await?; + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Rollback: Convert TEXT back to REAL (will cause precision loss again) + if manager.get_database_backend() == DatabaseBackend::Sqlite { + // Rollback accounts table + execute_sql( + manager, + r#" + CREATE TABLE accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, -- Back to REAL + discriminator TEXT, + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + INSERT INTO accounts_rollback + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, + CASE + WHEN lamports IS NOT NULL THEN CAST(lamports AS REAL) + ELSE NULL + END as lamports, + discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + DROP TABLE accounts; + ALTER TABLE accounts_rollback RENAME TO accounts; + + -- Recreate indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + + // Rollback token_accounts table + execute_sql( + manager, + r#" + CREATE TABLE token_accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + owner BLOB NOT NULL, + mint BLOB NOT NULL, + delegate BLOB, + state INTEGER NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + amount REAL, -- Back to REAL + tlv BLOB, + FOREIGN KEY (hash) REFERENCES accounts(hash) ON DELETE CASCADE + ); + + INSERT INTO token_accounts_rollback + SELECT + hash, owner, mint, delegate, state, spent, prev_spent, + CASE + WHEN amount IS NOT NULL THEN CAST(amount AS REAL) + ELSE NULL + END as amount, + tlv + FROM token_accounts; + + DROP TABLE token_accounts; + ALTER TABLE token_accounts_rollback RENAME TO token_accounts; + + -- Recreate indexes + CREATE UNIQUE INDEX token_accounts_owner_mint_hash_idx ON token_accounts (spent, owner, mint, hash); + CREATE UNIQUE INDEX token_accounts_delegate_mint_hash_idx ON token_accounts (spent, delegate, mint, hash); + "#, + ) + .await?; + + // Rollback owner_balances table + execute_sql( + manager, + r#" + CREATE TABLE owner_balances_rollback ( + owner BLOB NOT NULL PRIMARY KEY, + lamports REAL -- Back to REAL + ); + + INSERT INTO owner_balances_rollback + SELECT + owner, + CASE + WHEN lamports IS NOT NULL THEN CAST(lamports AS REAL) + ELSE 0 + END as lamports + FROM owner_balances; + + DROP TABLE owner_balances; + ALTER TABLE owner_balances_rollback RENAME TO owner_balances; + "#, + ) + .await?; + + // Rollback token_owner_balances table + execute_sql( + manager, + r#" + CREATE TABLE token_owner_balances_rollback ( + owner BLOB NOT NULL, + mint BLOB NOT NULL, + amount REAL, -- Back to REAL + PRIMARY KEY (owner, mint) + ); + + INSERT INTO token_owner_balances_rollback + SELECT + owner, mint, + CASE + WHEN amount IS NOT NULL THEN CAST(amount AS REAL) + ELSE 0 + END as amount + FROM token_owner_balances; + + DROP TABLE token_owner_balances; + ALTER TABLE token_owner_balances_rollback RENAME TO token_owner_balances; + "#, + ) + .await?; + } + + Ok(()) + } +} diff --git a/src/migration/migrations/standard/mod.rs b/src/migration/migrations/standard/mod.rs index 3bbc3c2d..32dd0852 100644 --- a/src/migration/migrations/standard/mod.rs +++ b/src/migration/migrations/standard/mod.rs @@ -9,6 +9,8 @@ pub mod m20241008_000006_init; pub mod m20250206_000007_init; pub mod m20250314_000008_init; pub mod m20250617_000152_fix_indexed_trees_unique_constraint; +pub mod m20250814_000009_fix_discriminator_precision; +pub mod m20250815_000010_fix_amounts_precision; pub fn get_standard_migrations() -> Vec> { vec![ @@ -21,5 +23,7 @@ pub fn get_standard_migrations() -> Vec> { Box::new(m20250206_000007_init::Migration), Box::new(m20250314_000008_init::Migration), Box::new(m20250617_000152_fix_indexed_trees_unique_constraint::Migration), + Box::new(m20250814_000009_fix_discriminator_precision::Migration), + Box::new(m20250815_000010_fix_amounts_precision::Migration), ] } diff --git a/test_discriminator_api.js b/test_discriminator_api.js new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/test_discriminator_api.js @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/integration_tests/mock_tests.rs b/tests/integration_tests/mock_tests.rs index 7f3347be..2ec78b19 100644 --- a/tests/integration_tests/mock_tests.rs +++ b/tests/integration_tests/mock_tests.rs @@ -1646,7 +1646,7 @@ async fn test_update_indexed_merkle_tree( tree, tree_type: TreeType::AddressV1, leaf: *indexed_element, - hash: Hash::new_unique().into(), // HACK: We don't care about the hash + hash: Hash::new_unique().into(), seq: *seq as u64, }, ); @@ -1671,3 +1671,210 @@ async fn test_update_indexed_merkle_tree( assert_eq!(tree_model.seq, Some(1i64)); } } + +// Discriminator serialization tests +#[tokio::test] +async fn test_discriminator_serializes_as_string() { + + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account_data = AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1, 2, 3]), + data_hash: Hash::default(), + }; + + + let json = serde_json::to_string(&account_data).unwrap(); + + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "Discriminator should be serialized as string, got: {}", + json + ); + + + assert!( + !json.contains(&format!("\"discriminator\":{}", expected_u64)), + "Discriminator should not be serialized as number, got: {}", + json + ); +} + +#[tokio::test] +async fn test_discriminator_prevents_javascript_precision_loss() { + use sqlx::types::Decimal; + + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let original_discriminator = u64::from_le_bytes(bytes); + + + let account_data = AccountData { + discriminator: UnsignedInteger(original_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", original_discriminator)), + "Discriminator should be serialized as string to prevent JS precision loss, got: {}", + json + ); + + assert!( + !json.contains(&format!("\"discriminator\":{}", original_discriminator)), + "Discriminator should not be serialized as number, got: {}", + json + ); + + let js_unsafe_value = 9007199254740993u64; // MAX_SAFE_INTEGER + 2 + + let unsafe_account_data = AccountData { + discriminator: UnsignedInteger(js_unsafe_value), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let unsafe_json = serde_json::to_string(&unsafe_account_data).unwrap(); + + assert!( + unsafe_json.contains(&format!("\"discriminator\":\"{}\"", js_unsafe_value)), + "JavaScript-unsafe discriminator should be serialized as string, got: {}", + unsafe_json + ); + + let decimal_val = Decimal::from(original_discriminator); + let fixed_parsed: u64 = decimal_val.try_into().unwrap(); + assert_eq!( + original_discriminator, fixed_parsed, + "Fixed conversion should preserve precision! Expected: {}, Got: {}", + original_discriminator, fixed_parsed + ); + + println!( + "SUCCESS: Discriminator serialized as string: \"{}\"", + original_discriminator + ); +} + +#[tokio::test] +async fn test_discriminator_with_max_u64() { + let max_discriminator = u64::MAX; + + let account_data = AccountData { + discriminator: UnsignedInteger(max_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), + "MAX u64 discriminator should be serialized as string, got: {}", + json + ); +} + +#[tokio::test] +async fn test_discriminator_zero_value() { + let account_data = AccountData { + discriminator: UnsignedInteger(0), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + assert!( + json.contains("\"discriminator\":\"0\""), + "Zero discriminator should be serialized as string, got: {}", + json + ); +} + +#[named] +#[rstest] +#[tokio::test] +#[serial] +async fn test_api_discriminator_serialization_integration( + #[values(DatabaseBackend::Sqlite, DatabaseBackend::Postgres)] db_backend: DatabaseBackend, +) { + let name = trim_test_name(function_name!()); + let setup = setup(name, db_backend).await; + + // HACK: We index a block so that API methods can fetch the current slot. + index_block( + &setup.db_conn, + &BlockInfo { + metadata: BlockMetadata { + slot: 0, + ..Default::default() + }, + ..Default::default() + }, + ) + .await + .unwrap(); + + let mut state_update = StateUpdate::new(); + + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account = Account { + hash: Hash::new_unique(), + address: Some(SerializablePubkey::new_unique()), + data: Some(AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1; 500]), + data_hash: Hash::new_unique(), + }), + owner: SerializablePubkey::new_unique(), + lamports: UnsignedInteger(1000), + tree: SerializablePubkey::new_unique(), + leaf_index: UnsignedInteger(0), + seq: Some(UnsignedInteger(0)), + slot_created: UnsignedInteger(0), + }; + + state_update.out_accounts.push(AccountWithContext { + account: account.clone(), + context: AccountContext::default(), + }); + persist_state_update_using_connection(&setup.db_conn, state_update) + .await + .unwrap(); + + let request = CompressedAccountRequest { + address: None, + hash: Some(account.hash.clone()), + }; + + let res_v1 = setup + .api + .get_compressed_account(request.clone()) + .await + .unwrap(); + + let json_v1 = serde_json::to_string(&res_v1).unwrap(); + assert!( + json_v1.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "V1 endpoint should serialize discriminator as string, got: {}", + json_v1 + ); + + let res_v2 = setup.api.get_compressed_account_v2(request).await.unwrap(); + + let json_v2 = serde_json::to_string(&res_v2).unwrap(); + assert!( + json_v2.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "V2 endpoint should serialize discriminator as string, got: {}", + json_v2 + ); +} diff --git a/tests/integration_tests/zeroeth_element_fix_test.rs b/tests/integration_tests/zeroeth_element_fix_test.rs index 6f2745bf..c75fe806 100644 --- a/tests/integration_tests/zeroeth_element_fix_test.rs +++ b/tests/integration_tests/zeroeth_element_fix_test.rs @@ -145,8 +145,6 @@ async fn test_reindex_fixes_wrong_zeroeth_element( let expected_zeroeth = get_zeroeth_exclusion_range_v1(tree_bytes.clone()); assert_eq!(fixed_zeroeth.next_index, expected_zeroeth.next_index); assert_eq!(fixed_zeroeth.value, expected_zeroeth.value); - - println!("✅ Re-indexing successfully fixed zeroeth element from wrong v2-style (next_index=0) to correct v1-style (next_index=1)"); } #[named] @@ -233,6 +231,4 @@ async fn test_reindex_preserves_correct_zeroeth_element( "Zeroeth element should still be correct" ); assert_eq!(zeroeth.seq, Some(1), "Seq should be updated"); - - println!("✅ Re-indexing preserves already correct zeroeth elements"); }