From fec6126ad8477d0b152d3d9ff2868d36c5639831 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Tue, 12 Aug 2025 12:11:36 -0400 Subject: [PATCH 01/15] add v2 tree and queue to parsing --- src/ingester/parser/tree_info.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ingester/parser/tree_info.rs b/src/ingester/parser/tree_info.rs index 3c073a7d..136b01a9 100644 --- a/src/ingester/parser/tree_info.rs +++ b/src/ingester/parser/tree_info.rs @@ -259,6 +259,10 @@ lazy_static! { pubkey!("bmt5yU97jC88YXTuSukYHa8Z5Bi2ZDUtmzfkDTA2mG2"), pubkey!("oq5oh5ZR3yGomuQgFduNDzjtGvVWfDRGLuDVjv9a96P"), ), + ( + pubkey!("2Yb3fGo2E9aWLjY8KuESaqurYpGGhEeJr7eynKrSgXwS"), + pubkey!("12wJT3xYd46rtjeqDU6CrtT8unqLjPiheggzqhN9YsyB"), + ), ]; let address_trees_v2 = [ From 65483bb17b45b9a34d55220a429583458667fec2 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Tue, 12 Aug 2025 13:38:40 -0400 Subject: [PATCH 02/15] use local crates --- Cargo.lock | 70 +++++++++++++++++++++++++------------- Cargo.toml | 11 ++++-- src/ingester/parser/mod.rs | 1 + 3 files changed, 55 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae825d0c..e646db51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,8 +105,7 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned-sized" version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48a526ec4434d531d488af59fe866f36b310fe8906691c75dffa664450a3800a" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "proc-macro2", "quote", @@ -3681,10 +3680,10 @@ dependencies = [ [[package]] name = "light-account-checks" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3fd000a2b8e0cc9d0b7b7712964870df51f2114f1693b9d8f0414f6f3ec16bd" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "solana-account-info", + "solana-msg", "solana-program-error", "solana-pubkey", "solana-sysvar", @@ -3694,15 +3693,14 @@ dependencies = [ [[package]] name = "light-batched-merkle-tree" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c7e179246468b09bf5c6882ef33043e178ff90eb6eab0c1c4c3623ef84b154" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "aligned-sized", "borsh 0.10.4", "light-account-checks", "light-bloom-filter", "light-compressed-account", - "light-hasher", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831)", "light-macros", "light-merkle-tree-metadata", "light-verifier", @@ -3719,8 +3717,7 @@ dependencies = [ [[package]] name = "light-bloom-filter" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44abcb5554e1c15cefa9ac17e4ceda6f5afb039db25ab1fd777f012356d0f964" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "bitvec", "num-bigint 0.4.6", @@ -3744,15 +3741,15 @@ dependencies = [ [[package]] name = "light-compressed-account" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f15113babaca9efb592631ec1e7e78c1c83413818a6e1e4248b7df53d88fe65" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "anchor-lang 0.31.1", "borsh 0.10.4", "bytemuck", - "light-hasher", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831)", "light-macros", "light-zero-copy", + "solana-msg", "solana-program-error", "solana-pubkey", "thiserror 2.0.12", @@ -3767,7 +3764,7 @@ checksum = "9b4f878301620df78ba7e7758c5fd720f28040f5c157375f88d310f15ddb1746" dependencies = [ "borsh 0.10.4", "light-bounded-vec", - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.9.1", "thiserror 2.0.12", ] @@ -3777,6 +3774,23 @@ name = "light-hasher" version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" +dependencies = [ + "ark-bn254 0.5.0", + "ark-ff 0.5.0", + "arrayvec", + "borsh 0.10.4", + "light-poseidon 0.3.0", + "num-bigint 0.4.6", + "sha2 0.10.9", + "sha3 0.10.8", + "solana-nostd-keccak", + "thiserror 2.0.12", +] + +[[package]] +name = "light-hasher" +version = "3.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3798,7 +3812,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc786d8df68ef64493fea04914a7a7745f8122f2efbae043cd4ba4eaffa9e6db" dependencies = [ - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.4.6", "num-traits", "thiserror 2.0.12", @@ -3807,8 +3821,7 @@ dependencies = [ [[package]] name = "light-macros" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "861c0817697c1201c2235cd831fcbaa2564a5f778e5229e9f5cc21035e97c273" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "bs58 0.5.1", "proc-macro2", @@ -3819,8 +3832,7 @@ dependencies = [ [[package]] name = "light-merkle-tree-metadata" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "544048fa95ea95fc1e952a2b9b1d6f09340c8decaffd1ad239fe1f6eb905ae76" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "borsh 0.10.4", "bytemuck", @@ -3838,7 +3850,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1650701feac958261b2c3ab4da361ad8548985ee3ee496a17e76db44d2d3c9e3" dependencies = [ - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "light-indexed-array", "num-bigint 0.4.6", "num-traits", @@ -3872,8 +3884,7 @@ dependencies = [ [[package]] name = "light-verifier" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fdf317ec3cfcd3a8e6556a5b5e7fbcc207a40264700f9a5271876838f26f58" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ "groth16-solana", "light-compressed-account", @@ -3883,14 +3894,25 @@ dependencies = [ [[package]] name = "light-zero-copy" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34d759f65547a6540db7047f38f4cb2c3f01658deca95a1dd06f26b578de947" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" dependencies = [ + "light-zero-copy-derive", "solana-program-error", "thiserror 2.0.12", "zerocopy", ] +[[package]] +name = "light-zero-copy-derive" +version = "0.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +dependencies = [ + "lazy_static", + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -4564,7 +4586,7 @@ dependencies = [ "light-batched-merkle-tree", "light-compressed-account", "light-concurrent-merkle-tree", - "light-hasher", + "light-hasher 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "light-merkle-tree-metadata", "light-merkle-tree-reference", "light-poseidon 0.3.0", diff --git a/Cargo.toml b/Cargo.toml index b0e81c70..79f612f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,9 +82,14 @@ solana-pubkey = "2.3.0" solana-transaction-status = "1.18.0" light-concurrent-merkle-tree = "2.1.0" -light-batched-merkle-tree = "0.3.0" -light-merkle-tree-metadata = "0.3.0" -light-compressed-account = { version = "0.3.0", features = ["anchor"] } +# light-batched-merkle-tree = "0.3.0" +# light-merkle-tree-metadata = "0.3.0" +# light-compressed-account = { version = "0.3.0", features = ["anchor"] } +light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } +light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } +light-compressed-account = { version = "0.3.0", features = [ + "anchor", +], git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } light-hasher = { version = "3.1.0" } light-poseidon = "0.3.0" diff --git a/src/ingester/parser/mod.rs b/src/ingester/parser/mod.rs index aedff22f..9f482707 100644 --- a/src/ingester/parser/mod.rs +++ b/src/ingester/parser/mod.rs @@ -65,6 +65,7 @@ pub fn parse_transaction(tx: &TransactionInfo, slot: u64) -> Result 1 { if get_compression_program_id() == instruction.program_id { From b7391565f944c4d995d6526b3f777aec5e08a4eb Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Wed, 13 Aug 2025 15:14:52 -0400 Subject: [PATCH 03/15] fix precision loss in parse_decimal --- src/api/method/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/method/utils.rs b/src/api/method/utils.rs index 6e532504..4bd974fd 100644 --- a/src/api/method/utils.rs +++ b/src/api/method/utils.rs @@ -29,9 +29,9 @@ use super::super::error::PhotonApiError; pub const PAGE_LIMIT: u64 = 1000; pub fn parse_decimal(value: Decimal) -> Result { + // Use try_into to avoid precision loss from string conversion value - .to_string() - .parse::() + .try_into() .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal value".to_string())) } From 0dca9f4e49ac2d37873122c33eb49b2dda7386c7 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Wed, 13 Aug 2025 17:57:27 -0400 Subject: [PATCH 04/15] add test --- .gitignore | 2 + src/common/typedefs/account/v1.rs | 108 ++++++++++++ tests/integration_tests/mock_tests.rs | 231 ++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) diff --git a/.gitignore b/.gitignore index 8196c671..324cf2b3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ test-ledger/ minio test.db docker-compose.yml + +photon.log \ No newline at end of file diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index 05d83051..92e87a75 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -29,11 +29,23 @@ pub struct Account { #[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema, Default)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AccountData { + #[serde(serialize_with = "serialize_discriminator_as_string")] pub discriminator: UnsignedInteger, pub data: Base64String, pub data_hash: Hash, } +// Fixes precision loss. +fn serialize_discriminator_as_string( + discriminator: &UnsignedInteger, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&discriminator.0.to_string()) +} + impl TryFrom for Account { type Error = PhotonApiError; @@ -70,3 +82,99 @@ impl TryFrom for Account { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_discriminator_serializes_as_string() { + // Test the discriminator value from the original precision loss issue + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account_data = AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1, 2, 3]), + data_hash: Hash::default(), + }; + + // Serialize to JSON + let json = serde_json::to_string(&account_data).unwrap(); + + // Verify discriminator is serialized as a string, not a number + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "Discriminator should be serialized as string, got: {}", + json + ); + + // Verify it doesn't contain the number format + assert!( + !json.contains(&format!("\"discriminator\":{}", expected_u64)), + "Discriminator should not be serialized as number, got: {}", + json + ); + } + + #[test] + fn test_discriminator_prevents_javascript_precision_loss() { + // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER + let large_discriminator = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 + + let account_data = AccountData { + discriminator: UnsignedInteger(large_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + // Should be serialized as string + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", large_discriminator)), + "Large discriminator should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } + + #[test] + fn test_discriminator_with_max_u64() { + // Test with u64::MAX + let max_discriminator = u64::MAX; + + let account_data = AccountData { + discriminator: UnsignedInteger(max_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + // Should be serialized as string + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), + "MAX u64 discriminator should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_discriminator_zero_value() { + // Test with zero value + let account_data = AccountData { + discriminator: UnsignedInteger(0), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + // Should be serialized as string "0" + assert!( + json.contains("\"discriminator\":\"0\""), + "Zero discriminator should be serialized as string, got: {}", + json + ); + } +} diff --git a/tests/integration_tests/mock_tests.rs b/tests/integration_tests/mock_tests.rs index 7f3347be..7498e924 100644 --- a/tests/integration_tests/mock_tests.rs +++ b/tests/integration_tests/mock_tests.rs @@ -1671,3 +1671,234 @@ async fn test_update_indexed_merkle_tree( assert_eq!(tree_model.seq, Some(1i64)); } } + +// Discriminator serialization tests +#[tokio::test] +async fn test_discriminator_serializes_as_string() { + // Test the discriminator value from the original precision loss issue + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account_data = AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1, 2, 3]), + data_hash: Hash::default(), + }; + + // Serialize to JSON + let json = serde_json::to_string(&account_data).unwrap(); + + // Verify discriminator is serialized as a string, not a number + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "Discriminator should be serialized as string, got: {}", + json + ); + + // Verify it doesn't contain the number format + assert!( + !json.contains(&format!("\"discriminator\":{}", expected_u64)), + "Discriminator should not be serialized as number, got: {}", + json + ); +} + +#[tokio::test] +async fn test_discriminator_prevents_javascript_precision_loss() { + use sqlx::types::Decimal; + + // Test with the original precision loss discriminator value from your issue + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let original_discriminator = u64::from_le_bytes(bytes); + println!("Original discriminator: {}", original_discriminator); + + // The precision loss you experienced was likely from JavaScript, not Rust + // But let's test that our string serialization prevents JavaScript precision loss + + // Test that our AccountData serializes the discriminator as a string + let account_data = AccountData { + discriminator: UnsignedInteger(original_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + println!("JSON output: {}", json); + + // Should be serialized as string to prevent JavaScript precision loss + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", original_discriminator)), + "Discriminator should be serialized as string to prevent JS precision loss, got: {}", + json + ); + + // Should NOT be serialized as a number + assert!( + !json.contains(&format!("\"discriminator\":{}", original_discriminator)), + "Discriminator should not be serialized as number, got: {}", + json + ); + + // Test with a value that definitely exceeds JavaScript's MAX_SAFE_INTEGER + let js_unsafe_value = 9007199254740993u64; // MAX_SAFE_INTEGER + 2 + println!("JavaScript unsafe value: {}", js_unsafe_value); + + let unsafe_account_data = AccountData { + discriminator: UnsignedInteger(js_unsafe_value), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let unsafe_json = serde_json::to_string(&unsafe_account_data).unwrap(); + println!("Unsafe JSON output: {}", unsafe_json); + + // This should also be serialized as string + assert!( + unsafe_json.contains(&format!("\"discriminator\":\"{}\"", js_unsafe_value)), + "JavaScript-unsafe discriminator should be serialized as string, got: {}", + unsafe_json + ); + + // Verify both conversions work correctly in Rust (the issue was client-side) + let decimal_val = Decimal::from(original_discriminator); + let fixed_parsed: u64 = decimal_val.try_into().unwrap(); + assert_eq!( + original_discriminator, fixed_parsed, + "Fixed conversion should preserve precision! Expected: {}, Got: {}", + original_discriminator, fixed_parsed + ); + + // The key fix: our discriminator is now serialized as a STRING in JSON + // This prevents JavaScript precision loss on the client side + println!( + "โœ… SUCCESS: Discriminator serialized as string: \"{}\"", + original_discriminator + ); + println!("โœ… This prevents JavaScript precision loss for values > MAX_SAFE_INTEGER"); +} + +#[tokio::test] +async fn test_discriminator_with_max_u64() { + // Test with u64::MAX + let max_discriminator = u64::MAX; + + let account_data = AccountData { + discriminator: UnsignedInteger(max_discriminator), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + // Should be serialized as string + assert!( + json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), + "MAX u64 discriminator should be serialized as string, got: {}", + json + ); +} + +#[tokio::test] +async fn test_discriminator_zero_value() { + // Test with zero value + let account_data = AccountData { + discriminator: UnsignedInteger(0), + data: Base64String(vec![]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + + // Should be serialized as string "0" + assert!( + json.contains("\"discriminator\":\"0\""), + "Zero discriminator should be serialized as string, got: {}", + json + ); +} + +#[named] +#[rstest] +#[tokio::test] +#[serial] +async fn test_api_discriminator_serialization_integration( + #[values(DatabaseBackend::Sqlite, DatabaseBackend::Postgres)] db_backend: DatabaseBackend, +) { + let name = trim_test_name(function_name!()); + let setup = setup(name, db_backend).await; + + // HACK: We index a block so that API methods can fetch the current slot. + index_block( + &setup.db_conn, + &BlockInfo { + metadata: BlockMetadata { + slot: 0, + ..Default::default() + }, + ..Default::default() + }, + ) + .await + .unwrap(); + + let mut state_update = StateUpdate::new(); + + // Test with the original precision loss discriminator value + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + + let account = Account { + hash: Hash::new_unique(), + address: Some(SerializablePubkey::new_unique()), + data: Some(AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1; 500]), + data_hash: Hash::new_unique(), + }), + owner: SerializablePubkey::new_unique(), + lamports: UnsignedInteger(1000), + tree: SerializablePubkey::new_unique(), + leaf_index: UnsignedInteger(0), + seq: Some(UnsignedInteger(0)), + slot_created: UnsignedInteger(0), + }; + + state_update.out_accounts.push(AccountWithContext { + account: account.clone(), + context: AccountContext::default(), + }); + persist_state_update_using_connection(&setup.db_conn, state_update) + .await + .unwrap(); + + let request = CompressedAccountRequest { + address: None, + hash: Some(account.hash.clone()), + }; + + // Test V1 endpoint + let res_v1 = setup + .api + .get_compressed_account(request.clone()) + .await + .unwrap(); + + // Serialize the response to JSON to verify discriminator format + let json_v1 = serde_json::to_string(&res_v1).unwrap(); + assert!( + json_v1.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "V1 endpoint should serialize discriminator as string, got: {}", + json_v1 + ); + + // Test V2 endpoint + let res_v2 = setup.api.get_compressed_account_v2(request).await.unwrap(); + + // Serialize the response to JSON to verify discriminator format + let json_v2 = serde_json::to_string(&res_v2).unwrap(); + assert!( + json_v2.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), + "V2 endpoint should serialize discriminator as string, got: {}", + json_v2 + ); +} From 2578c8109e1c24657618a279358b03955407bc68 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Wed, 13 Aug 2025 20:01:32 -0400 Subject: [PATCH 05/15] bump v --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 79f612f0..c1bc3ad5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.51.0" +version = "0.52.0" [[bin]] name = "photon" From ef6631133e61a7fd185b0cde33215b0523b44725 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Wed, 13 Aug 2025 20:39:58 -0400 Subject: [PATCH 06/15] 0.52.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/api/method/utils.rs | 14 ++ src/bin/test_discriminator_logging.rs | 41 +++++ src/common/typedefs/account/context.rs | 20 ++- src/common/typedefs/account/v1.rs | 12 +- src/common/typedefs/account/v2.rs | 4 +- src/dao/generated/accounts.rs | 3 +- src/ingester/persist/mod.rs | 14 +- ...0814_000009_fix_discriminator_precision.rs | 142 ++++++++++++++++++ src/migration/migrations/standard/mod.rs | 2 + test_discriminator_api.js | 1 + 12 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 src/bin/test_discriminator_logging.rs create mode 100644 src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs create mode 100644 test_discriminator_api.js diff --git a/Cargo.lock b/Cargo.lock index e646db51..29067b25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4555,7 +4555,7 @@ dependencies = [ [[package]] name = "photon-indexer" -version = "0.51.0" +version = "0.52.1" dependencies = [ "anchor-lang 0.29.0", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c1bc3ad5..01e382ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.52.0" +version = "0.52.1" [[bin]] name = "photon" diff --git a/src/api/method/utils.rs b/src/api/method/utils.rs index 4bd974fd..d47455b9 100644 --- a/src/api/method/utils.rs +++ b/src/api/method/utils.rs @@ -35,6 +35,20 @@ pub fn parse_decimal(value: Decimal) -> Result { .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal value".to_string())) } +pub fn parse_discriminator_string(value: String) -> Result { + // Parse discriminator from string to avoid precision loss + let parsed_u64 = value + .parse::() + .map_err(|_| PhotonApiError::UnexpectedError("Invalid discriminator string".to_string()))?; + + log::debug!( + "๐Ÿ“– DISCRIMINATOR RETRIEVED: string='{}' โ†’ u64={}", + value, + parsed_u64 + ); + Ok(parsed_u64) +} + pub(crate) fn parse_leaf_index(leaf_index: i64) -> Result { leaf_index .try_into() diff --git a/src/bin/test_discriminator_logging.rs b/src/bin/test_discriminator_logging.rs new file mode 100644 index 00000000..ea29a04b --- /dev/null +++ b/src/bin/test_discriminator_logging.rs @@ -0,0 +1,41 @@ +use photon_indexer::api::method::utils::parse_discriminator_string; +use photon_indexer::common::typedefs::account::AccountData; +use photon_indexer::common::typedefs::bs64_string::Base64String; +use photon_indexer::common::typedefs::hash::Hash; +use photon_indexer::common::typedefs::unsigned_integer::UnsignedInteger; + +fn main() { + // Initialize simple logging - just print to stdout for now + // env_logger not available, will use println! for demo + + println!("๐Ÿงช Testing discriminator logging flow..."); + + // Test the discriminator value from your original precision loss issue + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; + let expected_u64 = u64::from_le_bytes(bytes); + println!("Original discriminator: {}", expected_u64); + + // Simulate storage (string conversion) + let stored_as_string = expected_u64.to_string(); + println!("Stored as: '{}'", stored_as_string); + + // Simulate retrieval (string parsing) - this will log + println!("\n--- Testing retrieval logging ---"); + let retrieved_u64 = parse_discriminator_string(stored_as_string).unwrap(); + println!("Retrieved: {}", retrieved_u64); + + // Test JSON serialization logging + println!("\n--- Testing JSON serialization logging ---"); + let account_data = AccountData { + discriminator: UnsignedInteger(expected_u64), + data: Base64String(vec![1, 2, 3]), + data_hash: Hash::default(), + }; + + let json = serde_json::to_string(&account_data).unwrap(); + println!("JSON: {}", json); + + // Verify precision preserved + assert_eq!(expected_u64, retrieved_u64); + println!("\nโœ… Discriminator logging test completed successfully!"); +} diff --git a/src/common/typedefs/account/context.rs b/src/common/typedefs/account/context.rs index ac795925..f2403e61 100644 --- a/src/common/typedefs/account/context.rs +++ b/src/common/typedefs/account/context.rs @@ -1,5 +1,5 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::{parse_decimal, parse_leaf_index}; +use crate::api::method::utils::{parse_decimal, parse_discriminator_string, parse_leaf_index}; use crate::common::typedefs::account::{Account, AccountData}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; @@ -71,10 +71,18 @@ impl AccountWithContext { data, } = compressed_account; - let data = data.map(|d| AccountData { - discriminator: UnsignedInteger(LittleEndian::read_u64(&d.discriminator)), - data: Base64String(d.data), - data_hash: Hash::from(d.data_hash), + let data = data.map(|d| { + let discriminator_u64 = LittleEndian::read_u64(&d.discriminator); + log::debug!( + "๐Ÿ” DISCRIMINATOR INDEXED: bytes={:?} โ†’ u64={}", + d.discriminator, + discriminator_u64 + ); + AccountData { + discriminator: UnsignedInteger(discriminator_u64), + data: Base64String(d.data), + data_hash: Hash::from(d.data_hash), + } }); Self { @@ -111,7 +119,7 @@ impl TryFrom for AccountWithContext { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index 92e87a75..f27ac604 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -1,5 +1,5 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::parse_decimal; +use crate::api::method::utils::{parse_decimal, parse_discriminator_string}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; @@ -43,7 +43,13 @@ fn serialize_discriminator_as_string( where S: serde::Serializer, { - serializer.serialize_str(&discriminator.0.to_string()) + let discriminator_string = discriminator.0.to_string(); + log::debug!( + "๐Ÿ“ค DISCRIMINATOR RETURNED: u64={} โ†’ JSON string='{}'", + discriminator.0, + discriminator_string + ); + serializer.serialize_str(&discriminator_string) } impl TryFrom for Account { @@ -54,7 +60,7 @@ impl TryFrom for Account { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/account/v2.rs b/src/common/typedefs/account/v2.rs index 996d23b0..bc3319b4 100644 --- a/src/common/typedefs/account/v2.rs +++ b/src/common/typedefs/account/v2.rs @@ -1,6 +1,6 @@ use crate::api::error::PhotonApiError; use crate::api::method::get_validity_proof::MerkleContextV2; -use crate::api::method::utils::parse_decimal; +use crate::api::method::utils::{parse_decimal, parse_discriminator_string}; use crate::common::typedefs::account::AccountData; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; @@ -42,7 +42,7 @@ impl TryFrom for AccountV2 { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_decimal(discriminator)?), + discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/dao/generated/accounts.rs b/src/dao/generated/accounts.rs index c968a218..0dcb830c 100644 --- a/src/dao/generated/accounts.rs +++ b/src/dao/generated/accounts.rs @@ -19,8 +19,7 @@ pub struct Model { pub prev_spent: Option, #[sea_orm(column_type = "Decimal(Some((23, 0)))")] pub lamports: Decimal, - #[sea_orm(column_type = "Decimal(Some((23, 0)))", nullable)] - pub discriminator: Option, + pub discriminator: Option, pub tree_type: Option, pub nullified_in_tree: bool, pub nullifier_queue_index: Option, diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index d8f6a704..7d2cc7c9 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -413,11 +413,15 @@ async fn append_output_accounts( account_models.push(accounts::ActiveModel { hash: Set(account.account.hash.to_vec()), address: Set(account.account.address.map(|x| x.to_bytes_vec())), - discriminator: Set(account - .account - .data - .as_ref() - .map(|x| Decimal::from(x.discriminator.0))), + discriminator: Set(account.account.data.as_ref().map(|x| { + let discriminator_string = x.discriminator.0.to_string(); + log::debug!( + "๐Ÿ’พ DISCRIMINATOR STORED: u64={} โ†’ string='{}'", + x.discriminator.0, + discriminator_string + ); + discriminator_string + })), data: Set(account.account.data.as_ref().map(|x| x.data.clone().0)), data_hash: Set(account.account.data.as_ref().map(|x| x.data_hash.to_vec())), tree: Set(account.account.tree.to_bytes_vec()), diff --git a/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs b/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs new file mode 100644 index 00000000..733cd5bb --- /dev/null +++ b/src/migration/migrations/standard/m20250814_000009_fix_discriminator_precision.rs @@ -0,0 +1,142 @@ +use sea_orm_migration::{ + prelude::*, + sea_orm::{ConnectionTrait, DatabaseBackend, Statement}, +}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +async fn execute_sql<'a>(manager: &SchemaManager<'_>, sql: &str) -> Result<(), DbErr> { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + sql.to_string(), + )) + .await?; + Ok(()) +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Only apply this migration to SQLite - PostgreSQL already uses correct bigint2 type + if manager.get_database_backend() == DatabaseBackend::Sqlite { + execute_sql( + manager, + r#" + -- Fix discriminator precision loss by changing from REAL to TEXT + -- Step 1: Create new table with TEXT discriminator column + CREATE TABLE accounts_discriminator_fix ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, + discriminator TEXT, -- Changed from REAL to TEXT + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + -- Step 2: Copy data, converting REAL discriminator to TEXT + -- Note: This will lose precision for existing corrupted data, + -- but new data will be stored correctly as TEXT + INSERT INTO accounts_discriminator_fix + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, lamports, + CASE + WHEN discriminator IS NOT NULL THEN CAST(discriminator AS TEXT) + ELSE NULL + END as discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + -- Step 3: Drop old table and rename new one + DROP TABLE accounts; + ALTER TABLE accounts_discriminator_fix RENAME TO accounts; + + -- Step 4: Recreate all indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + + + "#, + ) + .await?; + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Rollback: Convert TEXT back to REAL (will cause precision loss again) + if manager.get_database_backend() == DatabaseBackend::Sqlite { + execute_sql( + manager, + r#" + -- Rollback discriminator from TEXT to REAL + CREATE TABLE accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, + discriminator REAL, -- Back to REAL + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + INSERT INTO accounts_rollback + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, lamports, + CASE + WHEN discriminator IS NOT NULL THEN CAST(discriminator AS REAL) + ELSE NULL + END as discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + DROP TABLE accounts; + ALTER TABLE accounts_rollback RENAME TO accounts; + + -- Recreate indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + } + + Ok(()) + } +} diff --git a/src/migration/migrations/standard/mod.rs b/src/migration/migrations/standard/mod.rs index 3bbc3c2d..58e11e62 100644 --- a/src/migration/migrations/standard/mod.rs +++ b/src/migration/migrations/standard/mod.rs @@ -9,6 +9,7 @@ pub mod m20241008_000006_init; pub mod m20250206_000007_init; pub mod m20250314_000008_init; pub mod m20250617_000152_fix_indexed_trees_unique_constraint; +pub mod m20250814_000009_fix_discriminator_precision; pub fn get_standard_migrations() -> Vec> { vec![ @@ -21,5 +22,6 @@ pub fn get_standard_migrations() -> Vec> { Box::new(m20250206_000007_init::Migration), Box::new(m20250314_000008_init::Migration), Box::new(m20250617_000152_fix_indexed_trees_unique_constraint::Migration), + Box::new(m20250814_000009_fix_discriminator_precision::Migration), ] } diff --git a/test_discriminator_api.js b/test_discriminator_api.js new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/test_discriminator_api.js @@ -0,0 +1 @@ + \ No newline at end of file From b3aec6b4e0797a89c8410203e8dad470f31c0d7a Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 18 Aug 2025 18:33:32 -0400 Subject: [PATCH 07/15] fix ingester for new ctoken --- Cargo.lock | 27 +++++++++++++-------------- Cargo.toml | 6 +++--- src/ingester/persist/mod.rs | 23 +++++++++++++++++------ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29067b25..7d818b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,7 +105,7 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned-sized" version = "1.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "proc-macro2", "quote", @@ -3680,7 +3680,7 @@ dependencies = [ [[package]] name = "light-account-checks" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "solana-account-info", "solana-msg", @@ -3693,14 +3693,14 @@ dependencies = [ [[package]] name = "light-batched-merkle-tree" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "aligned-sized", "borsh 0.10.4", "light-account-checks", "light-bloom-filter", "light-compressed-account", - "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831)", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=924289d77)", "light-macros", "light-merkle-tree-metadata", "light-verifier", @@ -3717,7 +3717,7 @@ dependencies = [ [[package]] name = "light-bloom-filter" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "bitvec", "num-bigint 0.4.6", @@ -3741,12 +3741,12 @@ dependencies = [ [[package]] name = "light-compressed-account" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "anchor-lang 0.31.1", "borsh 0.10.4", "bytemuck", - "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831)", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=924289d77)", "light-macros", "light-zero-copy", "solana-msg", @@ -3790,7 +3790,7 @@ dependencies = [ [[package]] name = "light-hasher" version = "3.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3821,7 +3821,7 @@ dependencies = [ [[package]] name = "light-macros" version = "2.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "bs58 0.5.1", "proc-macro2", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "light-merkle-tree-metadata" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "borsh 0.10.4", "bytemuck", @@ -3884,7 +3884,7 @@ dependencies = [ [[package]] name = "light-verifier" version = "2.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "groth16-solana", "light-compressed-account", @@ -3894,18 +3894,17 @@ dependencies = [ [[package]] name = "light-zero-copy" version = "0.2.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "light-zero-copy-derive", "solana-program-error", - "thiserror 2.0.12", "zerocopy", ] [[package]] name = "light-zero-copy-derive" version = "0.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=bce4d2d26fcd8536c32f157a2be33c4b644e9831#bce4d2d26fcd8536c32f157a2be33c4b644e9831" +source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "lazy_static", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 01e382ad..c2c3c41d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,11 +85,11 @@ light-concurrent-merkle-tree = "2.1.0" # light-batched-merkle-tree = "0.3.0" # light-merkle-tree-metadata = "0.3.0" # light-compressed-account = { version = "0.3.0", features = ["anchor"] } -light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } -light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } +light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } +light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } light-compressed-account = { version = "0.3.0", features = [ "anchor", -], git = "https://github.com/lightprotocol/light-protocol", rev = "bce4d2d26fcd8536c32f157a2be33c4b644e9831" } +], git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } light-hasher = { version = "3.1.0" } light-poseidon = "0.3.0" diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 7d2cc7c9..23b576c0 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -245,12 +245,23 @@ async fn persist_state_tree_history( pub fn parse_token_data(account: &Account) -> Result, IngesterError> { match account.data.clone() { - Some(data) if account.owner.0 == COMPRESSED_TOKEN_PROGRAM => { - let data_slice = data.data.0.as_slice(); - let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { - IngesterError::ParserError(format!("Failed to parse token data: {:?}", e)) - })?; - Ok(Some(token_data)) + Some(data) => { + let is_v1_token = data.discriminator.0.to_ne_bytes() == [2, 0, 0, 0, 0, 0, 0, 0]; // V1 discriminator + let is_v2_token = data.discriminator.0.to_ne_bytes() == [0, 0, 0, 0, 0, 0, 0, 3]; // V2 discriminator + println!("discriminator: {:?}", data.discriminator); + println!("discriminator.0: {:?}", data.discriminator.0); + println!("is_v1_token: {:?}", is_v1_token); + println!("is_v2_token: {:?}", is_v2_token); + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token) { + let data_slice = data.data.0.as_slice(); + let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { + IngesterError::ParserError(format!("Failed to parse token data: {:?}", e)) + })?; + Ok(Some(token_data)) + } else { + println!("Maybe mint account. address: {:?}", account.address); + Ok(None) + } } _ => Ok(None), } From 34b08a82ba51f8d658147eea87b0b19ea8fe5705 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 18 Aug 2025 18:41:45 -0400 Subject: [PATCH 08/15] bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c2c3c41d..ec4a64a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.52.1" +version = "0.52.2" [[bin]] name = "photon" From 889f35ac5260087869517ee6d365ef123bf4b903 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 18 Aug 2025 18:52:34 -0400 Subject: [PATCH 09/15] wip --- src/bin/test_discriminator_logging.rs | 10 ---------- src/common/typedefs/account/v1.rs | 5 ----- src/ingester/persist/mod.rs | 5 ----- tests/integration_tests/mock_tests.rs | 4 ++-- tests/integration_tests/zeroeth_element_fix_test.rs | 4 ---- 5 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/bin/test_discriminator_logging.rs b/src/bin/test_discriminator_logging.rs index ea29a04b..64d4e647 100644 --- a/src/bin/test_discriminator_logging.rs +++ b/src/bin/test_discriminator_logging.rs @@ -5,24 +5,16 @@ use photon_indexer::common::typedefs::hash::Hash; use photon_indexer::common::typedefs::unsigned_integer::UnsignedInteger; fn main() { - // Initialize simple logging - just print to stdout for now - // env_logger not available, will use println! for demo - - println!("๐Ÿงช Testing discriminator logging flow..."); - // Test the discriminator value from your original precision loss issue let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; let expected_u64 = u64::from_le_bytes(bytes); - println!("Original discriminator: {}", expected_u64); // Simulate storage (string conversion) let stored_as_string = expected_u64.to_string(); - println!("Stored as: '{}'", stored_as_string); // Simulate retrieval (string parsing) - this will log println!("\n--- Testing retrieval logging ---"); let retrieved_u64 = parse_discriminator_string(stored_as_string).unwrap(); - println!("Retrieved: {}", retrieved_u64); // Test JSON serialization logging println!("\n--- Testing JSON serialization logging ---"); @@ -33,9 +25,7 @@ fn main() { }; let json = serde_json::to_string(&account_data).unwrap(); - println!("JSON: {}", json); // Verify precision preserved assert_eq!(expected_u64, retrieved_u64); - println!("\nโœ… Discriminator logging test completed successfully!"); } diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index f27ac604..3532c574 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -44,11 +44,6 @@ where S: serde::Serializer, { let discriminator_string = discriminator.0.to_string(); - log::debug!( - "๐Ÿ“ค DISCRIMINATOR RETURNED: u64={} โ†’ JSON string='{}'", - discriminator.0, - discriminator_string - ); serializer.serialize_str(&discriminator_string) } diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 23b576c0..a7496d1c 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -426,11 +426,6 @@ async fn append_output_accounts( address: Set(account.account.address.map(|x| x.to_bytes_vec())), discriminator: Set(account.account.data.as_ref().map(|x| { let discriminator_string = x.discriminator.0.to_string(); - log::debug!( - "๐Ÿ’พ DISCRIMINATOR STORED: u64={} โ†’ string='{}'", - x.discriminator.0, - discriminator_string - ); discriminator_string })), data: Set(account.account.data.as_ref().map(|x| x.data.clone().0)), diff --git a/tests/integration_tests/mock_tests.rs b/tests/integration_tests/mock_tests.rs index 7498e924..4cdac8e1 100644 --- a/tests/integration_tests/mock_tests.rs +++ b/tests/integration_tests/mock_tests.rs @@ -1771,10 +1771,10 @@ async fn test_discriminator_prevents_javascript_precision_loss() { // The key fix: our discriminator is now serialized as a STRING in JSON // This prevents JavaScript precision loss on the client side println!( - "โœ… SUCCESS: Discriminator serialized as string: \"{}\"", + "SUCCESS: Discriminator serialized as string: \"{}\"", original_discriminator ); - println!("โœ… This prevents JavaScript precision loss for values > MAX_SAFE_INTEGER"); + println!("This prevents JavaScript precision loss for values > MAX_SAFE_INTEGER"); } #[tokio::test] diff --git a/tests/integration_tests/zeroeth_element_fix_test.rs b/tests/integration_tests/zeroeth_element_fix_test.rs index 6f2745bf..c75fe806 100644 --- a/tests/integration_tests/zeroeth_element_fix_test.rs +++ b/tests/integration_tests/zeroeth_element_fix_test.rs @@ -145,8 +145,6 @@ async fn test_reindex_fixes_wrong_zeroeth_element( let expected_zeroeth = get_zeroeth_exclusion_range_v1(tree_bytes.clone()); assert_eq!(fixed_zeroeth.next_index, expected_zeroeth.next_index); assert_eq!(fixed_zeroeth.value, expected_zeroeth.value); - - println!("โœ… Re-indexing successfully fixed zeroeth element from wrong v2-style (next_index=0) to correct v1-style (next_index=1)"); } #[named] @@ -233,6 +231,4 @@ async fn test_reindex_preserves_correct_zeroeth_element( "Zeroeth element should still be correct" ); assert_eq!(zeroeth.seq, Some(1), "Seq should be updated"); - - println!("โœ… Re-indexing preserves already correct zeroeth elements"); } From 72199d05912fd698435323f2e54f6ee9d417d8ef Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 18 Aug 2025 18:53:55 -0400 Subject: [PATCH 10/15] wip --- src/bin/test_discriminator_logging.rs | 31 -------------------------- src/common/typedefs/account/context.rs | 6 +---- 2 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 src/bin/test_discriminator_logging.rs diff --git a/src/bin/test_discriminator_logging.rs b/src/bin/test_discriminator_logging.rs deleted file mode 100644 index 64d4e647..00000000 --- a/src/bin/test_discriminator_logging.rs +++ /dev/null @@ -1,31 +0,0 @@ -use photon_indexer::api::method::utils::parse_discriminator_string; -use photon_indexer::common::typedefs::account::AccountData; -use photon_indexer::common::typedefs::bs64_string::Base64String; -use photon_indexer::common::typedefs::hash::Hash; -use photon_indexer::common::typedefs::unsigned_integer::UnsignedInteger; - -fn main() { - // Test the discriminator value from your original precision loss issue - let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; - let expected_u64 = u64::from_le_bytes(bytes); - - // Simulate storage (string conversion) - let stored_as_string = expected_u64.to_string(); - - // Simulate retrieval (string parsing) - this will log - println!("\n--- Testing retrieval logging ---"); - let retrieved_u64 = parse_discriminator_string(stored_as_string).unwrap(); - - // Test JSON serialization logging - println!("\n--- Testing JSON serialization logging ---"); - let account_data = AccountData { - discriminator: UnsignedInteger(expected_u64), - data: Base64String(vec![1, 2, 3]), - data_hash: Hash::default(), - }; - - let json = serde_json::to_string(&account_data).unwrap(); - - // Verify precision preserved - assert_eq!(expected_u64, retrieved_u64); -} diff --git a/src/common/typedefs/account/context.rs b/src/common/typedefs/account/context.rs index f2403e61..86c9ac7c 100644 --- a/src/common/typedefs/account/context.rs +++ b/src/common/typedefs/account/context.rs @@ -72,12 +72,8 @@ impl AccountWithContext { } = compressed_account; let data = data.map(|d| { + // TODO: check if v2 token account needs it differently. let discriminator_u64 = LittleEndian::read_u64(&d.discriminator); - log::debug!( - "๐Ÿ” DISCRIMINATOR INDEXED: bytes={:?} โ†’ u64={}", - d.discriminator, - discriminator_u64 - ); AccountData { discriminator: UnsignedInteger(discriminator_u64), data: Base64String(d.data), From dd268aad078fed10dd12011a40ba8d62474e4561 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Tue, 19 Aug 2025 09:00:42 -0400 Subject: [PATCH 11/15] add todo --- Cargo.lock | 2 +- src/ingester/persist/mod.rs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d818b64..202ab380 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4554,7 +4554,7 @@ dependencies = [ [[package]] name = "photon-indexer" -version = "0.52.1" +version = "0.52.2" dependencies = [ "anchor-lang 0.29.0", "anyhow", diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index a7496d1c..106a83ed 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -246,12 +246,15 @@ async fn persist_state_tree_history( pub fn parse_token_data(account: &Account) -> Result, IngesterError> { match account.data.clone() { Some(data) => { - let is_v1_token = data.discriminator.0.to_ne_bytes() == [2, 0, 0, 0, 0, 0, 0, 0]; // V1 discriminator - let is_v2_token = data.discriminator.0.to_ne_bytes() == [0, 0, 0, 0, 0, 0, 0, 3]; // V2 discriminator - println!("discriminator: {:?}", data.discriminator); - println!("discriminator.0: {:?}", data.discriminator.0); + let is_v1_token = data.discriminator.0.to_le_bytes() == [2, 0, 0, 0, 0, 0, 0, 0]; // V1 discriminator + let is_v2_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 3]; // V2 discriminator + + // TODO: remove after debugging. + println!("ingested data.discriminator: {:?}", data.discriminator); + println!("ingested data.discriminator.0: {:?}", data.discriminator.0); println!("is_v1_token: {:?}", is_v1_token); println!("is_v2_token: {:?}", is_v2_token); + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token) {} if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token) { let data_slice = data.data.0.as_slice(); let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { From 6ba68134350ea0fff9b49c3dcc0ba96705f9cbe8 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Tue, 19 Aug 2025 09:52:23 -0400 Subject: [PATCH 12/15] 0.52.3 --- Cargo.toml | 2 +- .../method/get_compressed_account_balance.rs | 3 +- .../method/get_compressed_balance_by_owner.rs | 2 +- .../get_compressed_mint_token_holders.rs | 3 +- .../get_compressed_token_account_balance.rs | 6 +- .../get_compressed_token_balances_by_owner.rs | 3 +- src/api/method/utils.rs | 28 +- src/common/typedefs/account/context.rs | 4 +- src/common/typedefs/account/v1.rs | 73 +++- src/common/typedefs/account/v2.rs | 7 +- src/common/typedefs/token_data.rs | 76 ++++- src/common/typedefs/unsigned_integer.rs | 12 + src/dao/generated/accounts.rs | 3 +- src/dao/generated/owner_balances.rs | 3 +- src/dao/generated/token_accounts.rs | 3 +- src/dao/generated/token_owner_balances.rs | 3 +- src/ingester/persist/mod.rs | 50 ++- .../m20250815_000010_fix_amounts_precision.rs | 318 ++++++++++++++++++ src/migration/migrations/standard/mod.rs | 2 + 19 files changed, 532 insertions(+), 69 deletions(-) create mode 100644 src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs diff --git a/Cargo.toml b/Cargo.toml index ec4a64a5..3d0da643 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.52.2" +version = "0.52.3" [[bin]] name = "photon" diff --git a/src/api/method/get_compressed_account_balance.rs b/src/api/method/get_compressed_account_balance.rs index 418d5c4b..9909ca32 100644 --- a/src/api/method/get_compressed_account_balance.rs +++ b/src/api/method/get_compressed_account_balance.rs @@ -5,7 +5,6 @@ use crate::common::typedefs::context::Context; use crate::common::typedefs::unsigned_integer::UnsignedInteger; use crate::dao::generated::accounts; use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect}; -use sqlx::types::Decimal; pub async fn get_compressed_account_balance( conn: &DatabaseConnection, @@ -22,7 +21,7 @@ pub async fn get_compressed_account_balance( .one(conn) .await? .map(|x| x.lamports) - .unwrap_or(Decimal::from(0)); + .unwrap_or_else(|| "0".to_string()); Ok(AccountBalanceResponse { value: UnsignedInteger(parse_decimal(balance)?), diff --git a/src/api/method/get_compressed_balance_by_owner.rs b/src/api/method/get_compressed_balance_by_owner.rs index e5ebfebd..5e0b06f1 100644 --- a/src/api/method/get_compressed_balance_by_owner.rs +++ b/src/api/method/get_compressed_balance_by_owner.rs @@ -28,7 +28,7 @@ pub async fn get_compressed_balance_by_owner( .into_model::() .all(conn) .await? - .iter() + .into_iter() .map(|x| parse_decimal(x.lamports)) .collect::, PhotonApiError>>()?; diff --git a/src/api/method/get_compressed_mint_token_holders.rs b/src/api/method/get_compressed_mint_token_holders.rs index c2ae1e98..ddea4edc 100644 --- a/src/api/method/get_compressed_mint_token_holders.rs +++ b/src/api/method/get_compressed_mint_token_holders.rs @@ -7,7 +7,7 @@ use crate::common::typedefs::bs58_string::Base58String; use crate::common::typedefs::context::Context; use crate::common::typedefs::limit::Limit; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_owner_balances; use super::super::error::PhotonApiError; @@ -16,6 +16,7 @@ use super::utils::{parse_decimal, PAGE_LIMIT}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct OwnerBalance { pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub balance: UnsignedInteger, } diff --git a/src/api/method/get_compressed_token_account_balance.rs b/src/api/method/get_compressed_token_account_balance.rs index a6fbbdd0..95fecd03 100644 --- a/src/api/method/get_compressed_token_account_balance.rs +++ b/src/api/method/get_compressed_token_account_balance.rs @@ -1,4 +1,4 @@ -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_accounts; use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect}; use serde::{Deserialize, Serialize}; @@ -7,7 +7,6 @@ use super::super::error::PhotonApiError; use super::utils::{parse_decimal, AccountDataTable}; use super::utils::{BalanceModel, CompressedAccountRequest}; use crate::common::typedefs::context::Context; -use sqlx::types::Decimal; use utoipa::ToSchema; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] @@ -15,6 +14,7 @@ use utoipa::ToSchema; // This is a struct because in the future we might add other fields here like decimals or uiAmount, // which is a string representation with decimals in the form of "10.00" pub struct TokenAccountBalance { + #[serde(serialize_with = "serialize_u64_as_string")] pub amount: UnsignedInteger, } @@ -40,7 +40,7 @@ pub async fn get_compressed_token_account_balance( .one(conn) .await? .map(|x| x.amount) - .unwrap_or(Decimal::from(0)); + .unwrap_or_else(|| "0".to_string()); Ok(GetCompressedTokenAccountBalanceResponse { value: TokenAccountBalance { diff --git a/src/api/method/get_compressed_token_balances_by_owner.rs b/src/api/method/get_compressed_token_balances_by_owner.rs index 4f888fd4..7e5cbf14 100644 --- a/src/api/method/get_compressed_token_balances_by_owner.rs +++ b/src/api/method/get_compressed_token_balances_by_owner.rs @@ -6,7 +6,7 @@ use crate::common::typedefs::bs58_string::Base58String; use crate::common::typedefs::context::Context; use crate::common::typedefs::limit::Limit; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::token_owner_balances; use super::super::error::PhotonApiError; @@ -15,6 +15,7 @@ use super::utils::{parse_decimal, PAGE_LIMIT}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct TokenBalance { pub mint: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub balance: UnsignedInteger, } diff --git a/src/api/method/utils.rs b/src/api/method/utils.rs index d47455b9..0c16d2a2 100644 --- a/src/api/method/utils.rs +++ b/src/api/method/utils.rs @@ -4,7 +4,7 @@ use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::serializable_signature::SerializableSignature; use crate::common::typedefs::token_data::{AccountState, TokenData}; use crate::common::typedefs::unix_timestamp::UnixTimestamp; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::{accounts, token_accounts}; use byteorder::{ByteOrder, LittleEndian}; @@ -28,24 +28,23 @@ use super::super::error::PhotonApiError; pub const PAGE_LIMIT: u64 = 1000; -pub fn parse_decimal(value: Decimal) -> Result { - // Use try_into to avoid precision loss from string conversion - value - .try_into() - .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal value".to_string())) +pub fn parse_decimal(value: String) -> Result { + // Parse from string to avoid precision loss + let parsed_u64 = value + .parse::() + .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal string".to_string()))?; + + log::debug!("value RETRIEVED: string='{}' โ†’ u64={}", value, parsed_u64); + Ok(parsed_u64) } -pub fn parse_discriminator_string(value: String) -> Result { +pub fn parse_u64_string(value: String) -> Result { // Parse discriminator from string to avoid precision loss let parsed_u64 = value .parse::() .map_err(|_| PhotonApiError::UnexpectedError("Invalid discriminator string".to_string()))?; - log::debug!( - "๐Ÿ“– DISCRIMINATOR RETRIEVED: string='{}' โ†’ u64={}", - value, - parsed_u64 - ); + log::debug!("value RETRIEVED: string='{}' โ†’ u64={}", value, parsed_u64); Ok(parsed_u64) } @@ -321,12 +320,12 @@ impl CompressedAccountRequest { #[derive(FromQueryResult)] pub struct BalanceModel { - pub amount: Decimal, + pub amount: String, } #[derive(FromQueryResult)] pub struct LamportModel { - pub lamports: Decimal, + pub lamports: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema, Default)] @@ -622,6 +621,7 @@ pub struct GetPaginatedSignaturesResponse { // We do not use generics to simplify documentation generation. pub struct AccountBalanceResponse { pub context: Context, + #[serde(serialize_with = "serialize_u64_as_string")] pub value: UnsignedInteger, } diff --git a/src/common/typedefs/account/context.rs b/src/common/typedefs/account/context.rs index 86c9ac7c..175b657f 100644 --- a/src/common/typedefs/account/context.rs +++ b/src/common/typedefs/account/context.rs @@ -1,5 +1,5 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::{parse_decimal, parse_discriminator_string, parse_leaf_index}; +use crate::api::method::utils::{parse_decimal, parse_leaf_index, parse_u64_string}; use crate::common::typedefs::account::{Account, AccountData}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; @@ -115,7 +115,7 @@ impl TryFrom for AccountWithContext { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index 3532c574..36cb17da 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -1,9 +1,9 @@ use crate::api::error::PhotonApiError; -use crate::api::method::utils::{parse_decimal, parse_discriminator_string}; +use crate::api::method::utils::{parse_decimal, parse_u64_string}; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::accounts::Model; use jsonrpsee_core::Serialize; use utoipa::ToSchema; @@ -15,6 +15,7 @@ pub struct Account { pub address: Option, pub data: Option, pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub lamports: UnsignedInteger, pub tree: SerializablePubkey, pub leaf_index: UnsignedInteger, @@ -29,24 +30,12 @@ pub struct Account { #[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema, Default)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AccountData { - #[serde(serialize_with = "serialize_discriminator_as_string")] + #[serde(serialize_with = "serialize_u64_as_string")] pub discriminator: UnsignedInteger, pub data: Base64String, pub data_hash: Hash, } -// Fixes precision loss. -fn serialize_discriminator_as_string( - discriminator: &UnsignedInteger, - serializer: S, -) -> Result -where - S: serde::Serializer, -{ - let discriminator_string = discriminator.0.to_string(); - serializer.serialize_str(&discriminator_string) -} - impl TryFrom for Account { type Error = PhotonApiError; @@ -55,7 +44,7 @@ impl TryFrom for Account { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { @@ -178,4 +167,56 @@ mod tests { json ); } + + #[test] + fn test_lamports_serializes_as_string() { + // Test that lamports field is serialized as string + let account = Account { + hash: Hash::default(), + address: None, + data: None, + owner: SerializablePubkey::default(), + lamports: UnsignedInteger(1000000000), // 1 SOL + tree: SerializablePubkey::default(), + leaf_index: UnsignedInteger(0), + seq: Some(UnsignedInteger(1)), + slot_created: UnsignedInteger(100), + }; + + let json = serde_json::to_string(&account).unwrap(); + + // Verify lamports is serialized as string + assert!( + json.contains("\"lamports\":\"1000000000\""), + "Lamports should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_lamports_prevents_javascript_precision_loss() { + // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER + let large_lamports = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 + + let account = Account { + hash: Hash::default(), + address: None, + data: None, + owner: SerializablePubkey::default(), + lamports: UnsignedInteger(large_lamports), + tree: SerializablePubkey::default(), + leaf_index: UnsignedInteger(0), + seq: None, + slot_created: UnsignedInteger(0), + }; + + let json = serde_json::to_string(&account).unwrap(); + + // Should be serialized as string + assert!( + json.contains(&format!("\"lamports\":\"{}\"", large_lamports)), + "Large lamports should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } } diff --git a/src/common/typedefs/account/v2.rs b/src/common/typedefs/account/v2.rs index bc3319b4..2e1a6aa8 100644 --- a/src/common/typedefs/account/v2.rs +++ b/src/common/typedefs/account/v2.rs @@ -1,11 +1,11 @@ use crate::api::error::PhotonApiError; use crate::api::method::get_validity_proof::MerkleContextV2; -use crate::api::method::utils::{parse_decimal, parse_discriminator_string}; +use crate::api::method::utils::{parse_decimal, parse_u64_string}; use crate::common::typedefs::account::AccountData; use crate::common::typedefs::bs64_string::Base64String; use crate::common::typedefs::hash::Hash; use crate::common::typedefs::serializable_pubkey::SerializablePubkey; -use crate::common::typedefs::unsigned_integer::UnsignedInteger; +use crate::common::typedefs::unsigned_integer::{serialize_u64_as_string, UnsignedInteger}; use crate::dao::generated::accounts::Model; use serde::Serialize; use utoipa::ToSchema; @@ -17,6 +17,7 @@ pub struct AccountV2 { pub address: Option, pub data: Option, pub owner: SerializablePubkey, + #[serde(serialize_with = "serialize_u64_as_string")] pub lamports: UnsignedInteger, pub leaf_index: UnsignedInteger, // For legacy trees is always Some() since the user tx appends directly to the Merkle tree @@ -42,7 +43,7 @@ impl TryFrom for AccountV2 { (Some(data), Some(data_hash), Some(discriminator)) => Some(AccountData { data: Base64String(data), data_hash: data_hash.try_into()?, - discriminator: UnsignedInteger(parse_discriminator_string(discriminator)?), + discriminator: UnsignedInteger(parse_u64_string(discriminator)?), }), (None, None, None) => None, _ => { diff --git a/src/common/typedefs/token_data.rs b/src/common/typedefs/token_data.rs index e7ad01b3..8d3154d5 100644 --- a/src/common/typedefs/token_data.rs +++ b/src/common/typedefs/token_data.rs @@ -5,7 +5,7 @@ use utoipa::ToSchema; use super::{ bs64_string::Base64String, serializable_pubkey::SerializablePubkey, - unsigned_integer::UnsignedInteger, + unsigned_integer::{serialize_u64_as_string, UnsignedInteger}, }; #[derive( @@ -40,6 +40,7 @@ pub struct TokenData { /// The owner of this account. pub owner: SerializablePubkey, /// The amount of tokens this account holds. + #[serde(serialize_with = "serialize_u64_as_string")] pub amount: UnsignedInteger, /// If `delegate` is `Some` then `delegated_amount` represents /// the amount authorized by the delegate @@ -49,3 +50,76 @@ pub struct TokenData { /// Placeholder for TokenExtension tlv data (unimplemented) pub tlv: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_amount_serializes_as_string() { + // Test that amount field is serialized as string + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(1000000000), // 1 token with 9 decimals + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + // Verify amount is serialized as string + assert!( + json.contains("\"amount\":\"1000000000\""), + "Amount should be serialized as string, got: {}", + json + ); + } + + #[test] + fn test_token_amount_prevents_javascript_precision_loss() { + // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER + let large_amount = u64::MAX; // Maximum u64 value + + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(large_amount), + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + // Should be serialized as string + assert!( + json.contains(&format!("\"amount\":\"{}\"", large_amount)), + "Large amount should be serialized as string to prevent JS precision loss, got: {}", + json + ); + } + + #[test] + fn test_token_amount_zero_value() { + // Test with zero value + let token_data = TokenData { + mint: SerializablePubkey::default(), + owner: SerializablePubkey::default(), + amount: UnsignedInteger(0), + delegate: None, + state: AccountState::initialized, + tlv: None, + }; + + let json = serde_json::to_string(&token_data).unwrap(); + + // Should be serialized as string "0" + assert!( + json.contains("\"amount\":\"0\""), + "Zero amount should be serialized as string, got: {}", + json + ); + } +} diff --git a/src/common/typedefs/unsigned_integer.rs b/src/common/typedefs/unsigned_integer.rs index 542fcd53..1d1f70bb 100644 --- a/src/common/typedefs/unsigned_integer.rs +++ b/src/common/typedefs/unsigned_integer.rs @@ -90,3 +90,15 @@ impl anchor_lang::AnchorSerialize for UnsignedInteger { writer.write_all(&self.0.to_le_bytes()) } } + +/// Serialize u64 as string to prevent precision loss using SQLite. +pub fn serialize_u64_as_string( + u64_value: &UnsignedInteger, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let u64_string = u64_value.0.to_string(); + serializer.serialize_str(&u64_string) +} diff --git a/src/dao/generated/accounts.rs b/src/dao/generated/accounts.rs index 0dcb830c..04cb19b0 100644 --- a/src/dao/generated/accounts.rs +++ b/src/dao/generated/accounts.rs @@ -17,8 +17,7 @@ pub struct Model { pub slot_created: i64, pub spent: bool, pub prev_spent: Option, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub lamports: Decimal, + pub lamports: String, pub discriminator: Option, pub tree_type: Option, pub nullified_in_tree: bool, diff --git a/src/dao/generated/owner_balances.rs b/src/dao/generated/owner_balances.rs index 2d4b20fb..ca359202 100644 --- a/src/dao/generated/owner_balances.rs +++ b/src/dao/generated/owner_balances.rs @@ -7,8 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub owner: Vec, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub lamports: Decimal, + pub lamports: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/dao/generated/token_accounts.rs b/src/dao/generated/token_accounts.rs index e9604cba..25df6d04 100644 --- a/src/dao/generated/token_accounts.rs +++ b/src/dao/generated/token_accounts.rs @@ -13,8 +13,7 @@ pub struct Model { pub state: i32, pub spent: bool, pub prev_spent: Option, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub amount: Decimal, + pub amount: String, pub tlv: Option>, } diff --git a/src/dao/generated/token_owner_balances.rs b/src/dao/generated/token_owner_balances.rs index 499c8518..f88f411b 100644 --- a/src/dao/generated/token_owner_balances.rs +++ b/src/dao/generated/token_owner_balances.rs @@ -9,8 +9,7 @@ pub struct Model { pub owner: Vec, #[sea_orm(primary_key, auto_increment = false)] pub mint: Vec, - #[sea_orm(column_type = "Decimal(Some((23, 0)))")] - pub amount: Decimal, + pub amount: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 106a83ed..720ecf59 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -342,14 +342,12 @@ async fn execute_account_update_query_and_update_balances( let prev_spent: Option = row.try_get("", "prev_spent")?; match (prev_spent, &modification_type) { (_, ModificationType::Append) | (Some(false), ModificationType::Spend) => { - let mut amount_of_interest = match db_backend { - DatabaseBackend::Postgres => row.try_get("", balance_column)?, - DatabaseBackend::Sqlite => { - let amount: i64 = row.try_get("", balance_column)?; - Decimal::from(amount) - } - _ => panic!("Unsupported database backend"), - }; + let amount_str: String = row.try_get("", balance_column)?; + let amount_u64 = amount_str.parse::().map_err(|_| { + IngesterError::DatabaseError(format!("Invalid amount string: {}", amount_str)) + })?; + let amount_i64 = amount_u64 as i64; + let mut amount_of_interest = Decimal::from(amount_i64); amount_of_interest *= multiplier; let owner = bytes_to_sql_format(db_backend, row.try_get("", "owner")?); let key = match account_type { @@ -373,16 +371,36 @@ async fn execute_account_update_query_and_update_balances( let values = balance_modifications .into_iter() .filter(|(_, value)| *value != Decimal::from(0)) - .map(|(key, value)| format!("({}, {})", key, value)) + .map(|(key, value)| { + if db_backend == DatabaseBackend::Sqlite { + // For SQLite, store as TEXT + let value_i64: i64 = value.try_into().unwrap_or(0); + let value_u64 = value_i64.unsigned_abs(); + format!("({}, '{}')", key, value_u64) + } else { + // For PostgreSQL, use numeric value + format!("({}, {})", key, value) + } + }) .collect::>(); if !values.is_empty() { let values_string = values.join(", "); - let raw_sql = format!( - "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) - VALUES {values_string} ON CONFLICT (owner{additional_columns}) - DO UPDATE SET {balance_column} = {owner_table_name}.{balance_column} + excluded.{balance_column}", - ); + let raw_sql = if txn.get_database_backend() == DatabaseBackend::Sqlite { + // For SQLite with TEXT columns, we need to cast to INTEGER for arithmetic + format!( + "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) + VALUES {values_string} ON CONFLICT (owner{additional_columns}) + DO UPDATE SET {balance_column} = CAST(CAST({owner_table_name}.{balance_column} AS INTEGER) + CAST(excluded.{balance_column} AS INTEGER) AS TEXT)", + ) + } else { + // PostgreSQL still uses numeric types + format!( + "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) + VALUES {values_string} ON CONFLICT (owner{additional_columns}) + DO UPDATE SET {balance_column} = {owner_table_name}.{balance_column} + excluded.{balance_column}", + ) + }; txn.execute(Statement::from_string(db_backend, raw_sql)) .await?; } @@ -442,7 +460,7 @@ async fn append_output_accounts( tree_type: Set(Some(account.context.tree_type as i32)), nullifier: Set(account.context.nullifier.as_ref().map(|x| x.to_vec())), owner: Set(account.account.owner.to_bytes_vec()), - lamports: Set(Decimal::from(account.account.lamports.0)), + lamports: Set(account.account.lamports.0.to_string()), spent: Set(false), slot_created: Set(account.account.slot_created.0 as i64), seq: Set(account.account.seq.map(|x| x.0 as i64)), @@ -494,7 +512,7 @@ pub async fn persist_token_accounts( hash: Set(hash.into()), mint: Set(token_data.mint.to_bytes_vec()), owner: Set(token_data.owner.to_bytes_vec()), - amount: Set(Decimal::from(token_data.amount.0)), + amount: Set(token_data.amount.0.to_string()), delegate: Set(token_data.delegate.map(|d| d.to_bytes_vec())), state: Set(token_data.state as i32), spent: Set(false), diff --git a/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs b/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs new file mode 100644 index 00000000..db1f74da --- /dev/null +++ b/src/migration/migrations/standard/m20250815_000010_fix_amounts_precision.rs @@ -0,0 +1,318 @@ +use sea_orm_migration::{ + prelude::*, + sea_orm::{ConnectionTrait, DatabaseBackend, Statement}, +}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +async fn execute_sql<'a>(manager: &SchemaManager<'_>, sql: &str) -> Result<(), DbErr> { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + sql.to_string(), + )) + .await?; + Ok(()) +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Only apply this migration to SQLite - PostgreSQL already uses correct bigint2 type + if manager.get_database_backend() == DatabaseBackend::Sqlite { + // Fix lamports in accounts table + execute_sql( + manager, + r#" + -- Fix lamports precision loss by changing from REAL to TEXT + -- Step 1: Create new table with TEXT lamports column + CREATE TABLE accounts_lamports_fix ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports TEXT, -- Changed from REAL to TEXT + discriminator TEXT, + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + -- Step 2: Copy data, converting REAL lamports to TEXT + INSERT INTO accounts_lamports_fix + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, + CASE + WHEN lamports IS NOT NULL THEN CAST(CAST(lamports AS INTEGER) AS TEXT) + ELSE NULL + END as lamports, + discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + -- Step 3: Drop old table and rename new one + DROP TABLE accounts; + ALTER TABLE accounts_lamports_fix RENAME TO accounts; + + -- Step 4: Recreate all indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + + // Fix amount in token_accounts table + execute_sql( + manager, + r#" + -- Fix amount precision loss in token_accounts + CREATE TABLE token_accounts_amount_fix ( + hash BLOB NOT NULL PRIMARY KEY, + owner BLOB NOT NULL, + mint BLOB NOT NULL, + delegate BLOB, + state INTEGER NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + amount TEXT, -- Changed from REAL to TEXT + tlv BLOB, + FOREIGN KEY (hash) REFERENCES accounts(hash) ON DELETE CASCADE + ); + + INSERT INTO token_accounts_amount_fix + SELECT + hash, owner, mint, delegate, state, spent, prev_spent, + CASE + WHEN amount IS NOT NULL THEN CAST(CAST(amount AS INTEGER) AS TEXT) + ELSE NULL + END as amount, + tlv + FROM token_accounts; + + DROP TABLE token_accounts; + ALTER TABLE token_accounts_amount_fix RENAME TO token_accounts; + + -- Recreate indexes + CREATE UNIQUE INDEX token_accounts_owner_mint_hash_idx ON token_accounts (spent, owner, mint, hash); + CREATE UNIQUE INDEX token_accounts_delegate_mint_hash_idx ON token_accounts (spent, delegate, mint, hash); + "#, + ) + .await?; + + // Fix lamports in owner_balances table + execute_sql( + manager, + r#" + -- Fix lamports precision loss in owner_balances + CREATE TABLE owner_balances_lamports_fix ( + owner BLOB NOT NULL PRIMARY KEY, + lamports TEXT -- Changed from REAL to TEXT + ); + + INSERT INTO owner_balances_lamports_fix + SELECT + owner, + CASE + WHEN lamports IS NOT NULL THEN CAST(CAST(lamports AS INTEGER) AS TEXT) + ELSE '0' + END as lamports + FROM owner_balances; + + DROP TABLE owner_balances; + ALTER TABLE owner_balances_lamports_fix RENAME TO owner_balances; + "#, + ) + .await?; + + // Fix amount in token_owner_balances table + execute_sql( + manager, + r#" + -- Fix amount precision loss in token_owner_balances + CREATE TABLE token_owner_balances_amount_fix ( + owner BLOB NOT NULL, + mint BLOB NOT NULL, + amount TEXT, -- Changed from REAL to TEXT + PRIMARY KEY (owner, mint) + ); + + INSERT INTO token_owner_balances_amount_fix + SELECT + owner, mint, + CASE + WHEN amount IS NOT NULL THEN CAST(CAST(amount AS INTEGER) AS TEXT) + ELSE '0' + END as amount + FROM token_owner_balances; + + DROP TABLE token_owner_balances; + ALTER TABLE token_owner_balances_amount_fix RENAME TO token_owner_balances; + "#, + ) + .await?; + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Rollback: Convert TEXT back to REAL (will cause precision loss again) + if manager.get_database_backend() == DatabaseBackend::Sqlite { + // Rollback accounts table + execute_sql( + manager, + r#" + CREATE TABLE accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + data BLOB, + data_hash BLOB, + address BLOB, + owner BLOB NOT NULL, + tree BLOB NOT NULL, + queue BLOB NULL, + leaf_index BIGINT NOT NULL, + seq BIGINT, + slot_created BIGINT NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + lamports REAL, -- Back to REAL + discriminator TEXT, + in_output_queue BOOLEAN NOT NULL DEFAULT TRUE, + nullifier BLOB, + tx_hash BLOB, + nullifier_queue_index BIGINT NULL, + nullified_in_tree BOOLEAN NOT NULL DEFAULT FALSE, + tree_type INTEGER NULL + ); + + INSERT INTO accounts_rollback + SELECT + hash, data, data_hash, address, owner, tree, queue, leaf_index, seq, + slot_created, spent, prev_spent, + CASE + WHEN lamports IS NOT NULL THEN CAST(lamports AS REAL) + ELSE NULL + END as lamports, + discriminator, + in_output_queue, nullifier, tx_hash, nullifier_queue_index, + nullified_in_tree, tree_type + FROM accounts; + + DROP TABLE accounts; + ALTER TABLE accounts_rollback RENAME TO accounts; + + -- Recreate indexes + CREATE INDEX accounts_address_spent_idx ON accounts (address, seq); + CREATE UNIQUE INDEX accounts_owner_hash_idx ON accounts (spent, owner, hash); + CREATE INDEX accounts_queue_idx ON accounts (tree, in_output_queue, leaf_index) WHERE in_output_queue = 1; + "#, + ) + .await?; + + // Rollback token_accounts table + execute_sql( + manager, + r#" + CREATE TABLE token_accounts_rollback ( + hash BLOB NOT NULL PRIMARY KEY, + owner BLOB NOT NULL, + mint BLOB NOT NULL, + delegate BLOB, + state INTEGER NOT NULL, + spent BOOLEAN NOT NULL, + prev_spent BOOLEAN, + amount REAL, -- Back to REAL + tlv BLOB, + FOREIGN KEY (hash) REFERENCES accounts(hash) ON DELETE CASCADE + ); + + INSERT INTO token_accounts_rollback + SELECT + hash, owner, mint, delegate, state, spent, prev_spent, + CASE + WHEN amount IS NOT NULL THEN CAST(amount AS REAL) + ELSE NULL + END as amount, + tlv + FROM token_accounts; + + DROP TABLE token_accounts; + ALTER TABLE token_accounts_rollback RENAME TO token_accounts; + + -- Recreate indexes + CREATE UNIQUE INDEX token_accounts_owner_mint_hash_idx ON token_accounts (spent, owner, mint, hash); + CREATE UNIQUE INDEX token_accounts_delegate_mint_hash_idx ON token_accounts (spent, delegate, mint, hash); + "#, + ) + .await?; + + // Rollback owner_balances table + execute_sql( + manager, + r#" + CREATE TABLE owner_balances_rollback ( + owner BLOB NOT NULL PRIMARY KEY, + lamports REAL -- Back to REAL + ); + + INSERT INTO owner_balances_rollback + SELECT + owner, + CASE + WHEN lamports IS NOT NULL THEN CAST(lamports AS REAL) + ELSE 0 + END as lamports + FROM owner_balances; + + DROP TABLE owner_balances; + ALTER TABLE owner_balances_rollback RENAME TO owner_balances; + "#, + ) + .await?; + + // Rollback token_owner_balances table + execute_sql( + manager, + r#" + CREATE TABLE token_owner_balances_rollback ( + owner BLOB NOT NULL, + mint BLOB NOT NULL, + amount REAL, -- Back to REAL + PRIMARY KEY (owner, mint) + ); + + INSERT INTO token_owner_balances_rollback + SELECT + owner, mint, + CASE + WHEN amount IS NOT NULL THEN CAST(amount AS REAL) + ELSE 0 + END as amount + FROM token_owner_balances; + + DROP TABLE token_owner_balances; + ALTER TABLE token_owner_balances_rollback RENAME TO token_owner_balances; + "#, + ) + .await?; + } + + Ok(()) + } +} diff --git a/src/migration/migrations/standard/mod.rs b/src/migration/migrations/standard/mod.rs index 58e11e62..32dd0852 100644 --- a/src/migration/migrations/standard/mod.rs +++ b/src/migration/migrations/standard/mod.rs @@ -10,6 +10,7 @@ pub mod m20250206_000007_init; pub mod m20250314_000008_init; pub mod m20250617_000152_fix_indexed_trees_unique_constraint; pub mod m20250814_000009_fix_discriminator_precision; +pub mod m20250815_000010_fix_amounts_precision; pub fn get_standard_migrations() -> Vec> { vec![ @@ -23,5 +24,6 @@ pub fn get_standard_migrations() -> Vec> { Box::new(m20250314_000008_init::Migration), Box::new(m20250617_000152_fix_indexed_trees_unique_constraint::Migration), Box::new(m20250814_000009_fix_discriminator_precision::Migration), + Box::new(m20250815_000010_fix_amounts_precision::Migration), ] } From 1f750db3929f097b0f6eb069b61254a77a839a8e Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Sun, 21 Sep 2025 14:29:16 -0400 Subject: [PATCH 13/15] local path, add shaFlat to tokendata parser --- Cargo.lock | 37 +++++++++++++++++++------------------ Cargo.toml | 16 +++++++++++----- src/ingester/persist/mod.rs | 13 ++++++++++--- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 202ab380..24cef932 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,7 +105,6 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned-sized" version = "1.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "proc-macro2", "quote", @@ -3680,7 +3679,6 @@ dependencies = [ [[package]] name = "light-account-checks" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "solana-account-info", "solana-msg", @@ -3693,14 +3691,13 @@ dependencies = [ [[package]] name = "light-batched-merkle-tree" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "aligned-sized", "borsh 0.10.4", "light-account-checks", "light-bloom-filter", "light-compressed-account", - "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=924289d77)", + "light-hasher 3.1.0", "light-macros", "light-merkle-tree-metadata", "light-verifier", @@ -3717,7 +3714,6 @@ dependencies = [ [[package]] name = "light-bloom-filter" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "bitvec", "num-bigint 0.4.6", @@ -3741,14 +3737,15 @@ dependencies = [ [[package]] name = "light-compressed-account" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "anchor-lang 0.31.1", "borsh 0.10.4", "bytemuck", - "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=924289d77)", + "light-hasher 3.1.0", "light-macros", + "light-profiler", "light-zero-copy", + "log", "solana-msg", "solana-program-error", "solana-pubkey", @@ -3772,8 +3769,6 @@ dependencies = [ [[package]] name = "light-hasher" version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3784,13 +3779,16 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.8", "solana-nostd-keccak", + "solana-program-error", + "solana-pubkey", "thiserror 2.0.12", ] [[package]] name = "light-hasher" version = "3.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3801,8 +3799,6 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.8", "solana-nostd-keccak", - "solana-program-error", - "solana-pubkey", "thiserror 2.0.12", ] @@ -3821,7 +3817,6 @@ dependencies = [ [[package]] name = "light-macros" version = "2.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "bs58 0.5.1", "proc-macro2", @@ -3832,7 +3827,6 @@ dependencies = [ [[package]] name = "light-merkle-tree-metadata" version = "0.3.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "borsh 0.10.4", "bytemuck", @@ -3881,10 +3875,18 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "light-profiler" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "light-verifier" version = "2.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "groth16-solana", "light-compressed-account", @@ -3894,8 +3896,8 @@ dependencies = [ [[package]] name = "light-zero-copy" version = "0.2.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ + "arrayvec", "light-zero-copy-derive", "solana-program-error", "zerocopy", @@ -3904,7 +3906,6 @@ dependencies = [ [[package]] name = "light-zero-copy-derive" version = "0.1.0" -source = "git+https://github.com/lightprotocol/light-protocol?rev=924289d77#924289d77917dd745d6331c5b97488e14d979de0" dependencies = [ "lazy_static", "proc-macro2", @@ -4554,7 +4555,7 @@ dependencies = [ [[package]] name = "photon-indexer" -version = "0.52.2" +version = "0.52.3" dependencies = [ "anchor-lang 0.29.0", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 3d0da643..ccc94fb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,14 +82,20 @@ solana-pubkey = "2.3.0" solana-transaction-status = "1.18.0" light-concurrent-merkle-tree = "2.1.0" + # light-batched-merkle-tree = "0.3.0" # light-merkle-tree-metadata = "0.3.0" # light-compressed-account = { version = "0.3.0", features = ["anchor"] } -light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } -light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } -light-compressed-account = { version = "0.3.0", features = [ - "anchor", -], git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } + +# light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } +# light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } +# light-compressed-account = { version = "0.3.0", features = [ "anchor"], git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } + +light-batched-merkle-tree = { path = "../light-protocol/program-libs/batched-merkle-tree" } +light-merkle-tree-metadata = { path = "../light-protocol/program-libs/merkle-tree-metadata" } +light-compressed-account = { features = ["anchor"], path = "../light-protocol/program-libs/compressed-account" } + + light-hasher = { version = "3.1.0" } light-poseidon = "0.3.0" diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 720ecf59..f7595db6 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -248,21 +248,28 @@ pub fn parse_token_data(account: &Account) -> Result, Ingester Some(data) => { let is_v1_token = data.discriminator.0.to_le_bytes() == [2, 0, 0, 0, 0, 0, 0, 0]; // V1 discriminator let is_v2_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 3]; // V2 discriminator + let is_sha_flat_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 4]; // V3 discriminator // TODO: remove after debugging. println!("ingested data.discriminator: {:?}", data.discriminator); - println!("ingested data.discriminator.0: {:?}", data.discriminator.0); println!("is_v1_token: {:?}", is_v1_token); println!("is_v2_token: {:?}", is_v2_token); + println!("is_sha_flat_token: {:?}", is_sha_flat_token); + println!("account.owner.0: {:?}", account.owner.0); if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token) {} - if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token) { + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token || is_sha_flat_token) { let data_slice = data.data.0.as_slice(); let token_data = TokenData::try_from_slice(data_slice).map_err(|e| { IngesterError::ParserError(format!("Failed to parse token data: {:?}", e)) })?; Ok(Some(token_data)) } else { - println!("Maybe mint account. address: {:?}", account.address); + if account.owner.0 == COMPRESSED_TOKEN_PROGRAM { + println!("Must be mint account. address: {:?}", account.address); + } + else { + println!("Not mint account. address: {:?}", account.address); + } Ok(None) } } From a0b540d908be66c835ecd84951857208a0e51345 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 22 Sep 2025 18:18:54 -0400 Subject: [PATCH 14/15] clean --- src/api/method/utils.rs | 18 +++++-------- src/common/typedefs/account/context.rs | 1 - src/common/typedefs/account/v1.rs | 17 ++---------- tests/integration_tests/mock_tests.rs | 36 +++++--------------------- 4 files changed, 14 insertions(+), 58 deletions(-) diff --git a/src/api/method/utils.rs b/src/api/method/utils.rs index 0c16d2a2..8948af3c 100644 --- a/src/api/method/utils.rs +++ b/src/api/method/utils.rs @@ -28,24 +28,18 @@ use super::super::error::PhotonApiError; pub const PAGE_LIMIT: u64 = 1000; +// Avoids precision loss pub fn parse_decimal(value: String) -> Result { - // Parse from string to avoid precision loss - let parsed_u64 = value + Ok(value .parse::() - .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal string".to_string()))?; - - log::debug!("value RETRIEVED: string='{}' โ†’ u64={}", value, parsed_u64); - Ok(parsed_u64) + .map_err(|_| PhotonApiError::UnexpectedError("Invalid decimal string".to_string()))?) } +// Avoids precision loss pub fn parse_u64_string(value: String) -> Result { - // Parse discriminator from string to avoid precision loss - let parsed_u64 = value + Ok(value .parse::() - .map_err(|_| PhotonApiError::UnexpectedError("Invalid discriminator string".to_string()))?; - - log::debug!("value RETRIEVED: string='{}' โ†’ u64={}", value, parsed_u64); - Ok(parsed_u64) + .map_err(|_| PhotonApiError::UnexpectedError("Invalid discriminator string".to_string()))?) } pub(crate) fn parse_leaf_index(leaf_index: i64) -> Result { diff --git a/src/common/typedefs/account/context.rs b/src/common/typedefs/account/context.rs index 175b657f..46c08fbc 100644 --- a/src/common/typedefs/account/context.rs +++ b/src/common/typedefs/account/context.rs @@ -72,7 +72,6 @@ impl AccountWithContext { } = compressed_account; let data = data.map(|d| { - // TODO: check if v2 token account needs it differently. let discriminator_u64 = LittleEndian::read_u64(&d.discriminator); AccountData { discriminator: UnsignedInteger(discriminator_u64), diff --git a/src/common/typedefs/account/v1.rs b/src/common/typedefs/account/v1.rs index 36cb17da..52a13c1e 100644 --- a/src/common/typedefs/account/v1.rs +++ b/src/common/typedefs/account/v1.rs @@ -79,7 +79,6 @@ mod tests { #[test] fn test_discriminator_serializes_as_string() { - // Test the discriminator value from the original precision loss issue let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; let expected_u64 = u64::from_le_bytes(bytes); @@ -89,17 +88,15 @@ mod tests { data_hash: Hash::default(), }; - // Serialize to JSON let json = serde_json::to_string(&account_data).unwrap(); - // Verify discriminator is serialized as a string, not a number assert!( json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), "Discriminator should be serialized as string, got: {}", json ); - // Verify it doesn't contain the number format + assert!( !json.contains(&format!("\"discriminator\":{}", expected_u64)), "Discriminator should not be serialized as number, got: {}", @@ -109,7 +106,6 @@ mod tests { #[test] fn test_discriminator_prevents_javascript_precision_loss() { - // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER let large_discriminator = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 let account_data = AccountData { @@ -120,7 +116,6 @@ mod tests { let json = serde_json::to_string(&account_data).unwrap(); - // Should be serialized as string assert!( json.contains(&format!("\"discriminator\":\"{}\"", large_discriminator)), "Large discriminator should be serialized as string to prevent JS precision loss, got: {}", @@ -130,7 +125,6 @@ mod tests { #[test] fn test_discriminator_with_max_u64() { - // Test with u64::MAX let max_discriminator = u64::MAX; let account_data = AccountData { @@ -141,7 +135,6 @@ mod tests { let json = serde_json::to_string(&account_data).unwrap(); - // Should be serialized as string assert!( json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), "MAX u64 discriminator should be serialized as string, got: {}", @@ -151,7 +144,6 @@ mod tests { #[test] fn test_discriminator_zero_value() { - // Test with zero value let account_data = AccountData { discriminator: UnsignedInteger(0), data: Base64String(vec![]), @@ -160,7 +152,6 @@ mod tests { let json = serde_json::to_string(&account_data).unwrap(); - // Should be serialized as string "0" assert!( json.contains("\"discriminator\":\"0\""), "Zero discriminator should be serialized as string, got: {}", @@ -170,13 +161,12 @@ mod tests { #[test] fn test_lamports_serializes_as_string() { - // Test that lamports field is serialized as string let account = Account { hash: Hash::default(), address: None, data: None, owner: SerializablePubkey::default(), - lamports: UnsignedInteger(1000000000), // 1 SOL + lamports: UnsignedInteger(1000000000), tree: SerializablePubkey::default(), leaf_index: UnsignedInteger(0), seq: Some(UnsignedInteger(1)), @@ -185,7 +175,6 @@ mod tests { let json = serde_json::to_string(&account).unwrap(); - // Verify lamports is serialized as string assert!( json.contains("\"lamports\":\"1000000000\""), "Lamports should be serialized as string, got: {}", @@ -195,7 +184,6 @@ mod tests { #[test] fn test_lamports_prevents_javascript_precision_loss() { - // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER let large_lamports = 9007199254740992u64; // MAX_SAFE_INTEGER + 1 let account = Account { @@ -212,7 +200,6 @@ mod tests { let json = serde_json::to_string(&account).unwrap(); - // Should be serialized as string assert!( json.contains(&format!("\"lamports\":\"{}\"", large_lamports)), "Large lamports should be serialized as string to prevent JS precision loss, got: {}", diff --git a/tests/integration_tests/mock_tests.rs b/tests/integration_tests/mock_tests.rs index 4cdac8e1..2ec78b19 100644 --- a/tests/integration_tests/mock_tests.rs +++ b/tests/integration_tests/mock_tests.rs @@ -1646,7 +1646,7 @@ async fn test_update_indexed_merkle_tree( tree, tree_type: TreeType::AddressV1, leaf: *indexed_element, - hash: Hash::new_unique().into(), // HACK: We don't care about the hash + hash: Hash::new_unique().into(), seq: *seq as u64, }, ); @@ -1675,7 +1675,7 @@ async fn test_update_indexed_merkle_tree( // Discriminator serialization tests #[tokio::test] async fn test_discriminator_serializes_as_string() { - // Test the discriminator value from the original precision loss issue + let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; let expected_u64 = u64::from_le_bytes(bytes); @@ -1685,17 +1685,17 @@ async fn test_discriminator_serializes_as_string() { data_hash: Hash::default(), }; - // Serialize to JSON + let json = serde_json::to_string(&account_data).unwrap(); - // Verify discriminator is serialized as a string, not a number + assert!( json.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), "Discriminator should be serialized as string, got: {}", json ); - // Verify it doesn't contain the number format + assert!( !json.contains(&format!("\"discriminator\":{}", expected_u64)), "Discriminator should not be serialized as number, got: {}", @@ -1707,15 +1707,10 @@ async fn test_discriminator_serializes_as_string() { async fn test_discriminator_prevents_javascript_precision_loss() { use sqlx::types::Decimal; - // Test with the original precision loss discriminator value from your issue let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; let original_discriminator = u64::from_le_bytes(bytes); - println!("Original discriminator: {}", original_discriminator); - // The precision loss you experienced was likely from JavaScript, not Rust - // But let's test that our string serialization prevents JavaScript precision loss - // Test that our AccountData serializes the discriminator as a string let account_data = AccountData { discriminator: UnsignedInteger(original_discriminator), data: Base64String(vec![]), @@ -1723,25 +1718,21 @@ async fn test_discriminator_prevents_javascript_precision_loss() { }; let json = serde_json::to_string(&account_data).unwrap(); - println!("JSON output: {}", json); - // Should be serialized as string to prevent JavaScript precision loss + assert!( json.contains(&format!("\"discriminator\":\"{}\"", original_discriminator)), "Discriminator should be serialized as string to prevent JS precision loss, got: {}", json ); - // Should NOT be serialized as a number assert!( !json.contains(&format!("\"discriminator\":{}", original_discriminator)), "Discriminator should not be serialized as number, got: {}", json ); - // Test with a value that definitely exceeds JavaScript's MAX_SAFE_INTEGER let js_unsafe_value = 9007199254740993u64; // MAX_SAFE_INTEGER + 2 - println!("JavaScript unsafe value: {}", js_unsafe_value); let unsafe_account_data = AccountData { discriminator: UnsignedInteger(js_unsafe_value), @@ -1750,16 +1741,13 @@ async fn test_discriminator_prevents_javascript_precision_loss() { }; let unsafe_json = serde_json::to_string(&unsafe_account_data).unwrap(); - println!("Unsafe JSON output: {}", unsafe_json); - // This should also be serialized as string assert!( unsafe_json.contains(&format!("\"discriminator\":\"{}\"", js_unsafe_value)), "JavaScript-unsafe discriminator should be serialized as string, got: {}", unsafe_json ); - // Verify both conversions work correctly in Rust (the issue was client-side) let decimal_val = Decimal::from(original_discriminator); let fixed_parsed: u64 = decimal_val.try_into().unwrap(); assert_eq!( @@ -1768,18 +1756,14 @@ async fn test_discriminator_prevents_javascript_precision_loss() { original_discriminator, fixed_parsed ); - // The key fix: our discriminator is now serialized as a STRING in JSON - // This prevents JavaScript precision loss on the client side println!( "SUCCESS: Discriminator serialized as string: \"{}\"", original_discriminator ); - println!("This prevents JavaScript precision loss for values > MAX_SAFE_INTEGER"); } #[tokio::test] async fn test_discriminator_with_max_u64() { - // Test with u64::MAX let max_discriminator = u64::MAX; let account_data = AccountData { @@ -1790,7 +1774,6 @@ async fn test_discriminator_with_max_u64() { let json = serde_json::to_string(&account_data).unwrap(); - // Should be serialized as string assert!( json.contains(&format!("\"discriminator\":\"{}\"", max_discriminator)), "MAX u64 discriminator should be serialized as string, got: {}", @@ -1800,7 +1783,6 @@ async fn test_discriminator_with_max_u64() { #[tokio::test] async fn test_discriminator_zero_value() { - // Test with zero value let account_data = AccountData { discriminator: UnsignedInteger(0), data: Base64String(vec![]), @@ -1809,7 +1791,6 @@ async fn test_discriminator_zero_value() { let json = serde_json::to_string(&account_data).unwrap(); - // Should be serialized as string "0" assert!( json.contains("\"discriminator\":\"0\""), "Zero discriminator should be serialized as string, got: {}", @@ -1843,7 +1824,6 @@ async fn test_api_discriminator_serialization_integration( let mut state_update = StateUpdate::new(); - // Test with the original precision loss discriminator value let bytes = [247u8, 237, 227, 245, 215, 195, 222, 70]; let expected_u64 = u64::from_le_bytes(bytes); @@ -1876,14 +1856,12 @@ async fn test_api_discriminator_serialization_integration( hash: Some(account.hash.clone()), }; - // Test V1 endpoint let res_v1 = setup .api .get_compressed_account(request.clone()) .await .unwrap(); - // Serialize the response to JSON to verify discriminator format let json_v1 = serde_json::to_string(&res_v1).unwrap(); assert!( json_v1.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), @@ -1891,10 +1869,8 @@ async fn test_api_discriminator_serialization_integration( json_v1 ); - // Test V2 endpoint let res_v2 = setup.api.get_compressed_account_v2(request).await.unwrap(); - // Serialize the response to JSON to verify discriminator format let json_v2 = serde_json::to_string(&res_v2).unwrap(); assert!( json_v2.contains(&format!("\"discriminator\":\"{}\"", expected_u64)), From 1ba1c07258885d4987b84ff9ba1a4f44cc616f81 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 22 Sep 2025 20:31:44 -0400 Subject: [PATCH 15/15] bump to 0.52.4 --- Cargo.lock | 24 ++++++++++++++++++------ Cargo.toml | 15 +++++++-------- src/common/typedefs/token_data.rs | 10 ++-------- src/common/typedefs/unsigned_integer.rs | 4 ++-- src/ingester/persist/mod.rs | 20 ++++++++++---------- 5 files changed, 39 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24cef932..a7cfd5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,6 +105,7 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned-sized" version = "1.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "proc-macro2", "quote", @@ -3679,6 +3680,7 @@ dependencies = [ [[package]] name = "light-account-checks" version = "0.3.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "solana-account-info", "solana-msg", @@ -3691,13 +3693,14 @@ dependencies = [ [[package]] name = "light-batched-merkle-tree" version = "0.3.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "aligned-sized", "borsh 0.10.4", "light-account-checks", "light-bloom-filter", "light-compressed-account", - "light-hasher 3.1.0", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db)", "light-macros", "light-merkle-tree-metadata", "light-verifier", @@ -3714,6 +3717,7 @@ dependencies = [ [[package]] name = "light-bloom-filter" version = "0.3.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "bitvec", "num-bigint 0.4.6", @@ -3737,11 +3741,12 @@ dependencies = [ [[package]] name = "light-compressed-account" version = "0.3.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "anchor-lang 0.31.1", "borsh 0.10.4", "bytemuck", - "light-hasher 3.1.0", + "light-hasher 3.1.0 (git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db)", "light-macros", "light-profiler", "light-zero-copy", @@ -3769,6 +3774,8 @@ dependencies = [ [[package]] name = "light-hasher" version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3779,16 +3786,13 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.8", "solana-nostd-keccak", - "solana-program-error", - "solana-pubkey", "thiserror 2.0.12", ] [[package]] name = "light-hasher" version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6445937ea244bebae0558e2aaec375791895d08c785b87cc45b62cd80d69139" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "ark-bn254 0.5.0", "ark-ff 0.5.0", @@ -3799,6 +3803,8 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.8", "solana-nostd-keccak", + "solana-program-error", + "solana-pubkey", "thiserror 2.0.12", ] @@ -3817,6 +3823,7 @@ dependencies = [ [[package]] name = "light-macros" version = "2.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "bs58 0.5.1", "proc-macro2", @@ -3827,6 +3834,7 @@ dependencies = [ [[package]] name = "light-merkle-tree-metadata" version = "0.3.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "borsh 0.10.4", "bytemuck", @@ -3878,6 +3886,7 @@ dependencies = [ [[package]] name = "light-profiler" version = "0.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "proc-macro2", "quote", @@ -3887,6 +3896,7 @@ dependencies = [ [[package]] name = "light-verifier" version = "2.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "groth16-solana", "light-compressed-account", @@ -3896,6 +3906,7 @@ dependencies = [ [[package]] name = "light-zero-copy" version = "0.2.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "arrayvec", "light-zero-copy-derive", @@ -3906,6 +3917,7 @@ dependencies = [ [[package]] name = "light-zero-copy-derive" version = "0.1.0" +source = "git+https://github.com/lightprotocol/light-protocol?rev=e3f0863db#e3f0863db7c85920a67b312f21e13bfd2b1d99b4" dependencies = [ "lazy_static", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index ccc94fb9..60a01ff8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ name = "photon-indexer" publish = true readme = "README.md" repository = "https://github.com/helius-labs/photon" -version = "0.52.3" +version = "0.52.4" [[bin]] name = "photon" @@ -87,17 +87,16 @@ light-concurrent-merkle-tree = "2.1.0" # light-merkle-tree-metadata = "0.3.0" # light-compressed-account = { version = "0.3.0", features = ["anchor"] } -# light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } -# light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } -# light-compressed-account = { version = "0.3.0", features = [ "anchor"], git = "https://github.com/lightprotocol/light-protocol", rev = "924289d77" } +light-batched-merkle-tree = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } +light-merkle-tree-metadata = { version = "0.3.0", git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } +light-compressed-account = { version = "0.3.0", features = [ "anchor"], git = "https://github.com/lightprotocol/light-protocol", rev = "e3f0863db" } -light-batched-merkle-tree = { path = "../light-protocol/program-libs/batched-merkle-tree" } -light-merkle-tree-metadata = { path = "../light-protocol/program-libs/merkle-tree-metadata" } -light-compressed-account = { features = ["anchor"], path = "../light-protocol/program-libs/compressed-account" } +# light-batched-merkle-tree = { path = "../light-protocol/program-libs/batched-merkle-tree" } +# light-merkle-tree-metadata = { path = "../light-protocol/program-libs/merkle-tree-metadata" } +# light-compressed-account = { features = ["anchor"], path = "../light-protocol/program-libs/compressed-account" } light-hasher = { version = "3.1.0" } - light-poseidon = "0.3.0" sqlx = { version = "0.6.2", features = [ diff --git a/src/common/typedefs/token_data.rs b/src/common/typedefs/token_data.rs index 8d3154d5..402a8136 100644 --- a/src/common/typedefs/token_data.rs +++ b/src/common/typedefs/token_data.rs @@ -57,11 +57,10 @@ mod tests { #[test] fn test_token_amount_serializes_as_string() { - // Test that amount field is serialized as string let token_data = TokenData { mint: SerializablePubkey::default(), owner: SerializablePubkey::default(), - amount: UnsignedInteger(1000000000), // 1 token with 9 decimals + amount: UnsignedInteger(1000000000), delegate: None, state: AccountState::initialized, tlv: None, @@ -69,7 +68,6 @@ mod tests { let json = serde_json::to_string(&token_data).unwrap(); - // Verify amount is serialized as string assert!( json.contains("\"amount\":\"1000000000\""), "Amount should be serialized as string, got: {}", @@ -79,8 +77,7 @@ mod tests { #[test] fn test_token_amount_prevents_javascript_precision_loss() { - // Test with a value that exceeds JavaScript's MAX_SAFE_INTEGER - let large_amount = u64::MAX; // Maximum u64 value + let large_amount = u64::MAX; let token_data = TokenData { mint: SerializablePubkey::default(), @@ -93,7 +90,6 @@ mod tests { let json = serde_json::to_string(&token_data).unwrap(); - // Should be serialized as string assert!( json.contains(&format!("\"amount\":\"{}\"", large_amount)), "Large amount should be serialized as string to prevent JS precision loss, got: {}", @@ -103,7 +99,6 @@ mod tests { #[test] fn test_token_amount_zero_value() { - // Test with zero value let token_data = TokenData { mint: SerializablePubkey::default(), owner: SerializablePubkey::default(), @@ -115,7 +110,6 @@ mod tests { let json = serde_json::to_string(&token_data).unwrap(); - // Should be serialized as string "0" assert!( json.contains("\"amount\":\"0\""), "Zero amount should be serialized as string, got: {}", diff --git a/src/common/typedefs/unsigned_integer.rs b/src/common/typedefs/unsigned_integer.rs index 1d1f70bb..9b2c8ec0 100644 --- a/src/common/typedefs/unsigned_integer.rs +++ b/src/common/typedefs/unsigned_integer.rs @@ -78,7 +78,7 @@ impl anchor_lang::AnchorDeserialize for UnsignedInteger { } fn deserialize_reader(reader: &mut R) -> Result { - let mut buffer = [0u8; 8]; // Adjusting the size for u64 + let mut buffer = [0u8; 8]; reader.read_exact(&mut buffer)?; let value = u64::from_le_bytes(buffer); Ok(UnsignedInteger(value)) @@ -91,7 +91,7 @@ impl anchor_lang::AnchorSerialize for UnsignedInteger { } } -/// Serialize u64 as string to prevent precision loss using SQLite. +/// Serialize u64 as string to prevent precision loss if using SQLite. pub fn serialize_u64_as_string( u64_value: &UnsignedInteger, serializer: S, diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index f7595db6..26d9cbea 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -251,11 +251,11 @@ pub fn parse_token_data(account: &Account) -> Result, Ingester let is_sha_flat_token = data.discriminator.0.to_le_bytes() == [0, 0, 0, 0, 0, 0, 0, 4]; // V3 discriminator // TODO: remove after debugging. - println!("ingested data.discriminator: {:?}", data.discriminator); - println!("is_v1_token: {:?}", is_v1_token); - println!("is_v2_token: {:?}", is_v2_token); - println!("is_sha_flat_token: {:?}", is_sha_flat_token); - println!("account.owner.0: {:?}", account.owner.0); + // println!("ingested data.discriminator: {:?}", data.discriminator); + // println!("is_v1_token: {:?}", is_v1_token); + // println!("is_v2_token: {:?}", is_v2_token); + // println!("is_sha_flat_token: {:?}", is_sha_flat_token); + // println!("account.owner.0: {:?}", account.owner.0); if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token) {} if account.owner.0 == COMPRESSED_TOKEN_PROGRAM && (is_v1_token || is_v2_token || is_sha_flat_token) { let data_slice = data.data.0.as_slice(); @@ -264,11 +264,12 @@ pub fn parse_token_data(account: &Account) -> Result, Ingester })?; Ok(Some(token_data)) } else { + // TODO: remove after debugging. if account.owner.0 == COMPRESSED_TOKEN_PROGRAM { - println!("Must be mint account. address: {:?}", account.address); + // println!("Must be mint account. address: {:?}", account.address); } else { - println!("Not mint account. address: {:?}", account.address); + // println!("Not mint account. address: {:?}", account.address); } Ok(None) } @@ -380,12 +381,12 @@ async fn execute_account_update_query_and_update_balances( .filter(|(_, value)| *value != Decimal::from(0)) .map(|(key, value)| { if db_backend == DatabaseBackend::Sqlite { - // For SQLite, store as TEXT + // Store as text let value_i64: i64 = value.try_into().unwrap_or(0); let value_u64 = value_i64.unsigned_abs(); format!("({}, '{}')", key, value_u64) } else { - // For PostgreSQL, use numeric value + // PostgreSQL format!("({}, {})", key, value) } }) @@ -401,7 +402,6 @@ async fn execute_account_update_query_and_update_balances( DO UPDATE SET {balance_column} = CAST(CAST({owner_table_name}.{balance_column} AS INTEGER) + CAST(excluded.{balance_column} AS INTEGER) AS TEXT)", ) } else { - // PostgreSQL still uses numeric types format!( "INSERT INTO {owner_table_name} (owner {additional_columns}, {balance_column}) VALUES {values_string} ON CONFLICT (owner{additional_columns})