From 9b02344ae9cebecba5441ffb09996c011211c51e Mon Sep 17 00:00:00 2001 From: Kyrylo Stepanov Date: Wed, 15 Jan 2025 16:47:17 +0200 Subject: [PATCH 1/5] fixed: add_filter_clause(), drop OR in favor of UNION clause --- postgre-client/src/asset_filter_client.rs | 71 ++++++++++++------- .../tests/asset_filter_client_test.rs | 71 ++++++++++++++++++- .../tests/asset_index_client_test.rs | 7 +- rocks-db/src/column.rs | 6 +- rocks-db/src/columns/asset.rs | 2 +- rocks-db/src/sequence_consistent.rs | 2 +- tests/setup/src/pg.rs | 33 ++++++--- 7 files changed, 145 insertions(+), 47 deletions(-) diff --git a/postgre-client/src/asset_filter_client.rs b/postgre-client/src/asset_filter_client.rs index cac11b6d..68177473 100644 --- a/postgre-client/src/asset_filter_client.rs +++ b/postgre-client/src/asset_filter_client.rs @@ -35,7 +35,7 @@ impl PgClient { options: &'a GetByMethodsOptions, ) -> Result<(QueryBuilder<'a, Postgres>, bool), IndexDbError> { let mut query_builder = QueryBuilder::new( - "SELECT ast_pubkey pubkey, ast_slot_created slot_created, ast_slot_updated slot_updated FROM assets_v3 ", + "(SELECT ast_pubkey pubkey, ast_slot_created slot_created, ast_slot_updated slot_updated FROM assets_v3 ", ); let group_clause_required = add_filter_clause(&mut query_builder, filter, options); @@ -102,8 +102,47 @@ impl PgClient { } // Add GROUP BY clause if necessary if group_clause_required { - query_builder.push(" GROUP BY assets_v3.ast_pubkey, assets_v3.ast_slot_created, assets_v3.ast_slot_updated "); + query_builder.push(" GROUP BY ast_pubkey, ast_slot_created, ast_slot_updated "); } + query_builder.push(") "); + + // the function with a side-effect that mutates the query_builder + if let Some(owner_address) = &filter.owner_address { + if let Some(ref token_type) = filter.token_type { + match *token_type { + TokenType::Fungible + | TokenType::NonFungible + | TokenType::RegularNFT + | TokenType::CompressedNFT => {}, + TokenType::All => { + query_builder.push(" UNION "); + query_builder.push(" ( "); + query_builder.push( + "SELECT + ast_pubkey, + ast_slot_created, + ast_slot_updated + FROM assets_v3 + JOIN fungible_tokens ON ast_pubkey = fungible_tokens.fbt_asset + WHERE ast_supply > 0 AND fbt_owner = ", + ); + query_builder.push_bind(owner_address); + if !options.show_zero_balance { + query_builder.push(" AND fbt_balance > "); + query_builder.push_bind(0i64); + } + if !options.show_unverified_collections { + // if there is no collection for asset it doesn't mean that it's unverified + query_builder.push( + " AND assets_v3.ast_is_collection_verified IS DISTINCT FROM FALSE", + ); + } + query_builder.push(")"); + }, + } + } + } + // Add ORDER BY clause let direction = match (&order.sort_direction, order_reversed) { (AssetSortDirection::Asc, true) | (AssetSortDirection::Desc, false) => " DESC ", @@ -111,10 +150,11 @@ impl PgClient { }; query_builder.push(" ORDER BY "); - query_builder.push(order.sort_by.to_string()); + query_builder.push(order.sort_by.to_string().replace("ast_", "")); query_builder.push(direction); - query_builder.push(", ast_pubkey "); + query_builder.push(", pubkey "); query_builder.push(direction); + // Add LIMIT clause query_builder.push(" LIMIT "); query_builder.push_bind(limit as i64); @@ -168,20 +208,6 @@ fn add_filter_clause<'a>( query_builder.push(" INNER JOIN assets_authorities ON assets_v3.ast_authority_fk = assets_authorities.auth_pubkey "); group_clause_required = true; } - if let Some(ref token_type) = filter.token_type { - if token_type == &TokenType::All && filter.owner_address.is_some() { - query_builder.push( - " LEFT JOIN fungible_tokens ON assets_v3.ast_pubkey = fungible_tokens.fbt_asset ", - ); - group_clause_required = true; - } - if token_type == &TokenType::Fungible && filter.owner_address.is_some() { - query_builder.push( - " INNER JOIN fungible_tokens ON assets_v3.ast_pubkey = fungible_tokens.fbt_asset ", - ); - group_clause_required = true; - } - } // todo: if we implement the additional params like negata and all/any switch, the true part and the AND prefix should be refactored query_builder.push(" WHERE TRUE "); @@ -259,14 +285,7 @@ fn add_filter_clause<'a>( TokenType::All => { query_builder.push(" AND (assets_v3.ast_owner = "); query_builder.push_bind(owner_address); - query_builder.push(" OR (fungible_tokens.fbt_owner = "); - query_builder.push_bind(owner_address); - if !options.show_zero_balance { - query_builder.push(" AND fungible_tokens.fbt_balance > "); - query_builder.push_bind(0i64); - } - query_builder.push(" ) "); - query_builder.push(" ) "); + query_builder.push(")"); }, } } else { diff --git a/postgre-client/tests/asset_filter_client_test.rs b/postgre-client/tests/asset_filter_client_test.rs index 6e7ee96a..a4ddb33d 100644 --- a/postgre-client/tests/asset_filter_client_test.rs +++ b/postgre-client/tests/asset_filter_client_test.rs @@ -1,7 +1,10 @@ #[cfg(feature = "integration_tests")] #[cfg(test)] mod tests { - use entities::{api_req_params::GetByMethodsOptions, enums::AssetType}; + use entities::{ + api_req_params::GetByMethodsOptions, + enums::{AssetType, TokenType}, + }; use postgre_client::{ model::*, storage_traits::{AssetIndexStorage, AssetPubkeyFilteredFetcher}, @@ -314,4 +317,70 @@ mod tests { assert_eq!(res.len(), 101); env.teardown().await; } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_search_for_fungible_asset() { + let cli = Cli::default(); + let env = TestEnvironment::new(&cli).await; + let storage = &env.client; + + // Generate nft asset + let asset_indexes = generate_asset_index_records(1); + let asset_index = &asset_indexes[0]; + let common_owner = asset_index.owner; + + // Generate fungible asset and make it part of the nft asset + let mut fungible_asset_indexes = generate_fungible_asset_index_records(1); + let fungible_asset_index = &mut fungible_asset_indexes[0]; + fungible_asset_index.pubkey = asset_index.pubkey; + fungible_asset_index.owner = common_owner; + + let last_known_key = generate_random_vec(8 + 8 + 32); + + // Insert assets and last key + storage.update_nft_asset_indexes_batch(asset_indexes.as_slice()).await.unwrap(); + storage.update_last_synced_key(&last_known_key, AssetType::NonFungible).await.unwrap(); + + // Insert fungible assets and last key + storage + .update_fungible_asset_indexes_batch(fungible_asset_indexes.as_slice()) + .await + .unwrap(); + storage.update_last_synced_key(&last_known_key, AssetType::Fungible).await.unwrap(); + + let cnt = env.count_rows_in_assets().await.unwrap(); + assert_eq!(cnt, 1); + let cnt = env.count_rows_in_fungible_tokens().await.unwrap(); + assert_eq!(cnt, 1); + + let order = AssetSorting { + sort_by: AssetSortBy::SlotCreated, + sort_direction: AssetSortDirection::Asc, + }; + + let res = storage + .get_asset_pubkeys_filtered( + &SearchAssetsFilter { + owner_address: Some(common_owner.unwrap().to_bytes().to_vec()), + token_type: Some(TokenType::All), + ..Default::default() + }, + &order, + 1, + None, + None, + None, + &GetByMethodsOptions { + show_zero_balance: true, + show_unverified_collections: true, + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(res.len(), 1); + + env.teardown().await; + } } diff --git a/postgre-client/tests/asset_index_client_test.rs b/postgre-client/tests/asset_index_client_test.rs index bfe5d2f9..1779dc31 100644 --- a/postgre-client/tests/asset_index_client_test.rs +++ b/postgre-client/tests/asset_index_client_test.rs @@ -8,6 +8,7 @@ mod tests { use postgre_client::storage_traits::AssetIndexStorage; use rand::Rng; use setup::pg::*; + use solana_sdk::pubkey::Pubkey; use testcontainers::clients::Cli; #[tokio::test] @@ -86,12 +87,12 @@ mod tests { // every asset_index will have 3 creators for asset_index in asset_indexes.iter_mut() { asset_index.creators.push(Creator { - creator: generate_random_pubkey(), + creator: Pubkey::new_unique(), creator_verified: rand::thread_rng().gen_bool(0.5), creator_share: 30, }); asset_index.creators.push(Creator { - creator: generate_random_pubkey(), + creator: Pubkey::new_unique(), creator_verified: rand::thread_rng().gen_bool(0.5), creator_share: 30, }); @@ -119,7 +120,7 @@ mod tests { .map(|asset_index| { let mut ai = asset_index.clone(); ai.creators.pop(); - ai.creators[1].creator = generate_random_pubkey(); + ai.creators[1].creator = Pubkey::new_unique(); ai.creators[0].creator_verified = !ai.creators[0].creator_verified; ai }) diff --git a/rocks-db/src/column.rs b/rocks-db/src/column.rs index 920b768b..94e8ad95 100644 --- a/rocks-db/src/column.rs +++ b/rocks-db/src/column.rs @@ -236,10 +236,8 @@ where ) .into_iter() .map(|res| { - res.map_err(StorageError::from).and_then(|opt| { - opt.map(|pinned| C::decode(pinned.as_ref()).map_err(StorageError::from)) - .transpose() - }) + res.map_err(StorageError::from) + .and_then(|opt| opt.map(|pinned| C::decode(pinned.as_ref())).transpose()) }) .collect() } diff --git a/rocks-db/src/columns/asset.rs b/rocks-db/src/columns/asset.rs index 3cf65c5a..5763fa9a 100644 --- a/rocks-db/src/columns/asset.rs +++ b/rocks-db/src/columns/asset.rs @@ -1958,7 +1958,7 @@ pub fn merge_complete_details_fb_simple_raw<'a>( } // Merge authority if let Some(new_authority) = new_val.authority() { - if authority.map_or(true, |current_authority| { + if authority.is_none_or(|current_authority| { new_authority.compare(¤t_authority) == Ordering::Greater }) { authority = Some(new_authority); diff --git a/rocks-db/src/sequence_consistent.rs b/rocks-db/src/sequence_consistent.rs index 4b1fc41b..706dcf69 100644 --- a/rocks-db/src/sequence_consistent.rs +++ b/rocks-db/src/sequence_consistent.rs @@ -30,7 +30,7 @@ impl SequenceConsistentManager for Storage { let result = if gap_found { self.trees_gaps.put_async(tree, TreesGaps {}).await } else { - self.trees_gaps.delete(tree).map_err(Into::into) + self.trees_gaps.delete(tree) }; if let Err(e) = result { error!("{} tree gap: {}", if gap_found { "Put" } else { "Delete" }, e); diff --git a/tests/setup/src/pg.rs b/tests/setup/src/pg.rs index a8e9261c..1e117889 100644 --- a/tests/setup/src/pg.rs +++ b/tests/setup/src/pg.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, path::PathBuf, str::FromStr, sync::Arc}; use entities::{ enums::*, - models::{AssetIndex, Creator, UrlWithStatus}, + models::{AssetIndex, Creator, FungibleAssetIndex, UrlWithStatus}, }; use metrics_utils::red::RequestErrorDurationMetrics; use postgre_client::{storage_traits::AssetIndexStorage, PgClient}; @@ -187,24 +187,20 @@ pub fn generate_random_vec(n: usize) -> Vec { random_vector } -pub fn generate_random_pubkey() -> Pubkey { - Pubkey::new_unique() -} - pub fn generate_asset_index_records(n: usize) -> Vec { let mut asset_indexes = Vec::new(); for i in 0..n { let asset_index = AssetIndex { - pubkey: generate_random_pubkey(), + pubkey: Pubkey::new_unique(), specification_version: SpecificationVersions::V1, specification_asset_class: SpecificationAssetClass::Nft, royalty_target_type: RoyaltyTargetType::Creators, royalty_amount: 1, slot_created: (n - i) as i64, - owner: Some(generate_random_pubkey()), - delegate: Some(generate_random_pubkey()), - authority: Some(generate_random_pubkey()), - collection: Some(generate_random_pubkey()), + owner: Some(Pubkey::new_unique()), + delegate: Some(Pubkey::new_unique()), + authority: Some(Pubkey::new_unique()), + collection: Some(Pubkey::new_unique()), is_collection_verified: Some(rand::thread_rng().gen_bool(0.5)), is_burnt: false, is_compressible: false, @@ -218,7 +214,7 @@ pub fn generate_asset_index_records(n: usize) -> Vec { update_authority: None, slot_updated: (n + 10 + i) as i64, creators: vec![Creator { - creator: generate_random_pubkey(), + creator: Pubkey::new_unique(), creator_verified: rand::thread_rng().gen_bool(0.5), creator_share: 100, }], @@ -230,3 +226,18 @@ pub fn generate_asset_index_records(n: usize) -> Vec { } asset_indexes } + +pub fn generate_fungible_asset_index_records(n: usize) -> Vec { + let mut asset_indexes = Vec::new(); + for i in 0..n { + let asset_index = FungibleAssetIndex { + pubkey: Pubkey::new_unique(), + owner: Some(Pubkey::new_unique()), + slot_updated: (n + 10 + i) as i64, + fungible_asset_mint: None, + fungible_asset_balance: Some(1), + }; + asset_indexes.push(asset_index); + } + asset_indexes +} From d6eed9c2fb09fa105650ef0c3fe4cbccc57495ad Mon Sep 17 00:00:00 2001 From: n00m4d Date: Thu, 16 Jan 2025 15:18:44 +0100 Subject: [PATCH 2/5] feat: add condition to join with fungible tokens table --- ...YNZgmq8B1wbDzm2SZvvSeKULXaPBPp9HDkvGtcccZP | Bin 0 -> 824 bytes ...iDhCQMDzxj8NcLfRBCQj3R9mQkE1DnDZfrNbAgruQk | Bin 0 -> 312 bytes ...zjtWZcZyvADaT5rrkRwGKWjnuzvK3PDedGMUwpnrrP | Bin 0 -> 312 bytes ...MfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG | Bin 0 -> 224 bytes ...U9EA8ApkD3eCYjoR3e8MkJzvjcVb8nTFUQhGKMjA7r | Bin 0 -> 312 bytes ...eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E | Bin 0 -> 312 bytes ...L8nj9woKgjsGE11BJ7J4BFoo9SgmLn7vYu3Ek3mPEy | Bin 0 -> 824 bytes ...QXpnPDt6NazCuFeXZxBYcuL46gsBBJ4CPqmcgkg3Hd | Bin 0 -> 824 bytes ...jgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj | Bin 0 -> 224 bytes ...hWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK | Bin 0 -> 224 bytes ...FxPHhiQWptvrc2YA2HyfNjCvqxpsEtmjmREBvWf7NJ | Bin 0 -> 312 bytes ...MfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG | 1 + ...jgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj | Bin 0 -> 32 bytes ...hWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK | Bin 0 -> 32 bytes integration_tests/src/mpl_core_tests.rs | 1 - integration_tests/src/regular_nft_tests.rs | 56 ++++++++++++++++++ postgre-client/src/asset_filter_client.rs | 2 +- 17 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3LYNZgmq8B1wbDzm2SZvvSeKULXaPBPp9HDkvGtcccZP create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3oiDhCQMDzxj8NcLfRBCQj3R9mQkE1DnDZfrNbAgruQk create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3rzjtWZcZyvADaT5rrkRwGKWjnuzvK3PDedGMUwpnrrP create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/6kU9EA8ApkD3eCYjoR3e8MkJzvjcVb8nTFUQhGKMjA7r create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/94eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/AZL8nj9woKgjsGE11BJ7J4BFoo9SgmLn7vYu3Ek3mPEy create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/AvQXpnPDt6NazCuFeXZxBYcuL46gsBBJ4CPqmcgkg3Hd create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK create mode 100644 integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/sFxPHhiQWptvrc2YA2HyfNjCvqxpsEtmjmREBvWf7NJ create mode 100644 integration_tests/src/data/largest_token_account_ids/3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG/3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG create mode 100644 integration_tests/src/data/largest_token_account_ids/BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj/BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj create mode 100644 integration_tests/src/data/largest_token_account_ids/HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK/HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3LYNZgmq8B1wbDzm2SZvvSeKULXaPBPp9HDkvGtcccZP b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3LYNZgmq8B1wbDzm2SZvvSeKULXaPBPp9HDkvGtcccZP new file mode 100644 index 0000000000000000000000000000000000000000..8c634066760760225b5a445b0f495d2fb3ade10f GIT binary patch literal 824 zcmY#jfB*@G90nE!4+a$=H-NzfNJap00)qsQRNC=yNrguK{@>?sZoACLD7D6~J4huu zL+xv$?lp(N7%uLD)QyiX*0@^C4XV#(Iea!^M^3TSg1=imw_V)B{6o`C17zlwP$0>` z@E-~o!8`^ASBJ{{DNxRGCI$u;RrwFrms2LO?6F#`b0+FT?d4B*OqM-mi!Q4-6fwFI5w d0;zEfaD|AUK$jlkd`8Cqu+SaiJ{iQ$2LNh4Q!M}h literal 0 HcmV?d00001 diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3oiDhCQMDzxj8NcLfRBCQj3R9mQkE1DnDZfrNbAgruQk b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3oiDhCQMDzxj8NcLfRBCQj3R9mQkE1DnDZfrNbAgruQk new file mode 100644 index 0000000000000000000000000000000000000000..0d7c734a8222e456727ba0128ba1093521904b1b GIT binary patch literal 312 zcmY#jfB*@G90nE!4+a$=H-NzfNJap00)qsQ)LgfzZB6Yp;WJA*r(KxTv@|}5ox$Tm z?InZk$jnAS(|f z85sUU0V9~l!0^DKGJgt`vlJ*cBieDl=h`R+*H0ah+6F694?3u@uUOpk{mF-mUyhV> y{8E+wV0}4d63ZT|#X4uAKGa_RbjM`bQ?}@`dPAP6waEf7bD_rLrASeWZV&)E$xulE literal 0 HcmV?d00001 diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3rzjtWZcZyvADaT5rrkRwGKWjnuzvK3PDedGMUwpnrrP b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/3rzjtWZcZyvADaT5rrkRwGKWjnuzvK3PDedGMUwpnrrP new file mode 100644 index 0000000000000000000000000000000000000000..000f201b86a0aa6ea88e688d9bc081d5470d489e GIT binary patch literal 312 zcmY#jfB*@G90nE!4+a$=H-NzfNJap00)qsQ)M}Gka=154;7I`A-IA9lcYu4Pd{`!_quY8%$C--@zLfJt$Mz;*8OK#sR147(pxshNVEPqwZ>u&&hY7)Ux;QZjp@z1rNU5$q%fbcdoK} zis7bz$5rJ&SYJ+=#Ina~vCf&O54D#+-7#7Alr6ff-jHW%ZL$E^AR-BcYu4Pd{`!_quY8%$C--@zLfJt$Mz;*8OK#sR1XDLwZ_y2u?4~qp8R`uc^yi-fgD`(a(d~MnCs~e%89(lnm1lXuRnY`B9X|zg1AbN5 FWdS8CRYd>* literal 0 HcmV?d00001 diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/94eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/94eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E new file mode 100644 index 0000000000000000000000000000000000000000..2140b7ddb5ffa9a587f543ce6bb95367f0001f5f GIT binary patch literal 312 zcmY#jfB*@G90nE!4+a$=H-NzfNJap00)qsQEI%`O=3TWq|L6zR_a2{ry!>^&pq`n> z72aFZp55G`_obKZ?ze~6Qx{IYdHSK-x!09zWVW=vjgK~;Xw~zzweCN|N)4bHA7tf$ zBm={LD0slJR|v%Cv9HXZ!U$q9Ff0Y)-~aaoJ}eeYSk<>vX8&W^-t@hy=MTLTVw~O= zBehxS&ly$u57w7cCb8_XTC8&>>O<}2Pj^g~J!Ok7t2gACTAM5YHi$?9YAunfQI!Dz DcW+hN literal 0 HcmV?d00001 diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/AZL8nj9woKgjsGE11BJ7J4BFoo9SgmLn7vYu3Ek3mPEy b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/AZL8nj9woKgjsGE11BJ7J4BFoo9SgmLn7vYu3Ek3mPEy new file mode 100644 index 0000000000000000000000000000000000000000..d166b05870069c5c88af12af85c9e7160477226a GIT binary patch literal 824 zcmY#jfB*@G90nE!4+a$=H-NzfNJap00)qsQ>|;!K*Js&yU!(O@_`c@_L7{g<@9MMq z#_gJG+|%zCoWxy_y7BSF8drCYJjpurv$MfnOE;rT_m8TmP> gnoz3{6c9-6(!L*%bN z_f>v@SH--7|G#h-q;7nCvBuS6Zcu$T%i*&TJ93Jp7X01nx$WW}<{z4F8Xz;bgaSzh zhW}8&2<9;`+;gnVp91A9XJTMrQI-E-eK}dleojtma!F=>9>i8IkeXmWX9({Ekjf}2DJZtm*DnU?ECy*zhU?MG%P&bRN=?iu rEy~eH63!}ynFldvFpwz>jG(|I6o?Ed3=E9_Kq?s-tOtX;hnyP#{$qj) literal 0 HcmV?d00001 diff --git a/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj b/integration_tests/src/data/accounts/search_by_owner_with_show_zero_balance/BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj new file mode 100644 index 0000000000000000000000000000000000000000..07aaeb94f3031724055d03a76a209eb93b693508 GIT binary patch literal 224 zcmY#jfB*@G3 z`Tpd?#V<$7IexL-{r2#B>cYu4Pd{`!_quY8%$C--@zLfJt$Mz;*8OK#2{JRmR}4rp zF#LxCMlg?o;kJEc{uCf52*?MC+3e9dI;Z3Lgp0FU7h29%dla}q_%5SgSdj8Nm zA;#%_F;bhA{+waE`|aWN)P<98o_^?d?serFnJukvCb?f3pE}uMT%NKFZI@GWA{)53v^lbo~$h)EAzYsA1 literal 0 HcmV?d00001 diff --git a/integration_tests/src/mpl_core_tests.rs b/integration_tests/src/mpl_core_tests.rs index 84b67f5c..1136e460 100644 --- a/integration_tests/src/mpl_core_tests.rs +++ b/integration_tests/src/mpl_core_tests.rs @@ -67,7 +67,6 @@ async fn test_mpl_core_get_collection() { } #[tokio::test] -#[ignore = "TODO: must recheck snapshot incompatibility"] #[serial] #[named] async fn test_mpl_core_get_assets_by_authority() { diff --git a/integration_tests/src/regular_nft_tests.rs b/integration_tests/src/regular_nft_tests.rs index df9bd0ad..cc61f871 100644 --- a/integration_tests/src/regular_nft_tests.rs +++ b/integration_tests/src/regular_nft_tests.rs @@ -164,3 +164,59 @@ async fn test_reg_search_assets() { let response = setup.das_api.search_assets(request, mutexed_tasks.clone()).await.unwrap(); insta::assert_json_snapshot!(name, response); } + +#[tokio::test] +#[serial] +#[named] +async fn test_search_by_owner_with_show_zero_balance() { + let name = trim_test_name(function_name!()); + let setup = TestSetup::new_with_options( + name.clone(), + TestSetupOptions { + network: Some(Network::Mainnet), + clear_db: true, + }, + ) + .await; + + let seeds: Vec = seed_token_mints([ + "HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK" // mint for fungible acc + ]); + index_seed_events(&setup, seeds.iter().collect_vec()).await; + + let seeds: Vec = seed_accounts([ + "3rzjtWZcZyvADaT5rrkRwGKWjnuzvK3PDedGMUwpnrrP", // empty token acc from NFT (3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG) + "94eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E", // fungible token with zero balance + ]); + + index_seed_events(&setup, seeds.iter().collect_vec()).await; + + let seeds: Vec = seed_nfts([ + "BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj", // NFT wallet has + "3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG" // NFT wallet used to have + ]); + + index_seed_events(&setup, seeds.iter().collect_vec()).await; + + let request = r#" + { + "page": 1, + "limit": 500, + "ownerAddress": "3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM", + "tokenType": "all", + "options": { + "showNativeBalance": true, "showZeroBalance": true + } + } + "#; + + let mutexed_tasks = Arc::new(Mutex::new(JoinSet::new())); + + let request: SearchAssets = serde_json::from_str(request).unwrap(); + let response = setup + .das_api + .search_assets(request, mutexed_tasks.clone()) + .await + .unwrap(); + insta::assert_json_snapshot!(name, response); +} diff --git a/postgre-client/src/asset_filter_client.rs b/postgre-client/src/asset_filter_client.rs index 68177473..0a139851 100644 --- a/postgre-client/src/asset_filter_client.rs +++ b/postgre-client/src/asset_filter_client.rs @@ -123,7 +123,7 @@ impl PgClient { ast_slot_created, ast_slot_updated FROM assets_v3 - JOIN fungible_tokens ON ast_pubkey = fungible_tokens.fbt_asset + JOIN fungible_tokens ON ast_pubkey = fungible_tokens.fbt_asset AND ast_owner_type = 'token' WHERE ast_supply > 0 AND fbt_owner = ", ); query_builder.push_bind(owner_address); From 96a03b29821e4418df2d25e82cce0f9d2e94e97b Mon Sep 17 00:00:00 2001 From: n00m4d Date: Mon, 20 Jan 2025 12:59:40 +0100 Subject: [PATCH 3/5] chore: fmt --- integration_tests/src/regular_nft_tests.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/integration_tests/src/regular_nft_tests.rs b/integration_tests/src/regular_nft_tests.rs index cc61f871..316b2a21 100644 --- a/integration_tests/src/regular_nft_tests.rs +++ b/integration_tests/src/regular_nft_tests.rs @@ -172,15 +172,12 @@ async fn test_search_by_owner_with_show_zero_balance() { let name = trim_test_name(function_name!()); let setup = TestSetup::new_with_options( name.clone(), - TestSetupOptions { - network: Some(Network::Mainnet), - clear_db: true, - }, + TestSetupOptions { network: Some(Network::Mainnet), clear_db: true }, ) .await; let seeds: Vec = seed_token_mints([ - "HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK" // mint for fungible acc + "HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK", // mint for fungible acc ]); index_seed_events(&setup, seeds.iter().collect_vec()).await; @@ -193,7 +190,7 @@ async fn test_search_by_owner_with_show_zero_balance() { let seeds: Vec = seed_nfts([ "BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj", // NFT wallet has - "3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG" // NFT wallet used to have + "3yMfqHsajYFw2Yw6C4kwrvHRESMg9U7isNVJuzNETJKG", // NFT wallet used to have ]); index_seed_events(&setup, seeds.iter().collect_vec()).await; @@ -213,10 +210,6 @@ async fn test_search_by_owner_with_show_zero_balance() { let mutexed_tasks = Arc::new(Mutex::new(JoinSet::new())); let request: SearchAssets = serde_json::from_str(request).unwrap(); - let response = setup - .das_api - .search_assets(request, mutexed_tasks.clone()) - .await - .unwrap(); + let response = setup.das_api.search_assets(request, mutexed_tasks.clone()).await.unwrap(); insta::assert_json_snapshot!(name, response); } From 82bb1996526e7c665374fb304813cc1cc5ccaa74 Mon Sep 17 00:00:00 2001 From: n00m4d Date: Mon, 20 Jan 2025 13:28:56 +0100 Subject: [PATCH 4/5] test: add snapshot --- ...earch_by_owner_with_show_zero_balance.snap | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap diff --git a/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap b/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap new file mode 100644 index 00000000..87658896 --- /dev/null +++ b/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap @@ -0,0 +1,147 @@ +--- +source: integration_tests/src/regular_nft_tests.rs +assertion_line: 214 +expression: response +snapshot_kind: text +--- +{ + "total": 2, + "limit": 500, + "page": 1, + "items": [ + { + "interface": "Custom", + "id": "HxhWkVpk5NS4Ltg5nij2G671CKXFRKPK8vy271Ub4uEK", + "content": { + "$schema": "https://schema.metaplex.com/nft1.0.json", + "json_uri": "", + "files": [], + "metadata": { + "name": "Hxro (Wormhole)", + "symbol": "HXRO" + }, + "links": {} + }, + "authorities": [ + { + "address": "BCD75RNBHrJJpW4dXVagL5mPjzRLnVZq4YirJdjEYMV7", + "scopes": [ + "full" + ] + } + ], + "compression": { + "eligible": false, + "compressed": false, + "data_hash": "", + "creator_hash": "", + "asset_hash": "", + "tree": "", + "seq": 0, + "leaf_id": 0 + }, + "grouping": [], + "royalty": { + "royalty_model": "creators", + "target": null, + "percent": 0.0, + "basis_points": 0, + "primary_sale_happened": false, + "locked": false + }, + "creators": [], + "ownership": { + "frozen": false, + "delegated": false, + "delegate": null, + "ownership_model": "single", + "owner": "3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM" + }, + "supply": null, + "mutable": true, + "burnt": false, + "lamports": 5616720, + "executable": false, + "metadata_owner": "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", + "rent_epoch": 18446744073709551615, + "token_info": { + "balance": 0, + "supply": 69421234056477841, + "decimals": 8, + "token_program": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "associated_token_address": "94eSnb5qBWTvxj3gqP6Ukq8bPhRTNNVZrE7zR5yTZd9E", + "mint_authority": "BCD75RNBHrJJpW4dXVagL5mPjzRLnVZq4YirJdjEYMV7" + } + }, + { + "interface": "V1_NFT", + "id": "BFjgKzLNKZEbZoDrESi79ai8jXgyBth1HXCJPXBGs8sj", + "content": { + "$schema": "https://schema.metaplex.com/nft1.0.json", + "json_uri": "", + "files": [], + "metadata": { + "name": "Degen Ape", + "symbol": "DAPE", + "token_standard": "NonFungible" + }, + "links": {} + }, + "authorities": [ + { + "address": "3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM", + "scopes": [ + "full" + ] + } + ], + "compression": { + "eligible": false, + "compressed": false, + "data_hash": "", + "creator_hash": "", + "asset_hash": "", + "tree": "", + "seq": 0, + "leaf_id": 0 + }, + "grouping": [], + "royalty": { + "royalty_model": "creators", + "target": null, + "percent": 0.0, + "basis_points": 0, + "primary_sale_happened": false, + "locked": false + }, + "creators": [], + "ownership": { + "frozen": false, + "delegated": false, + "delegate": null, + "ownership_model": "single", + "owner": "3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM" + }, + "supply": { + "print_max_supply": 0, + "print_current_supply": 0, + "edition_nonce": 255 + }, + "mutable": true, + "burnt": false, + "lamports": 5616720, + "executable": false, + "metadata_owner": "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", + "rent_epoch": 18446744073709551615, + "token_info": { + "balance": 1, + "supply": 1, + "decimals": 0, + "token_program": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "associated_token_address": "3oiDhCQMDzxj8NcLfRBCQj3R9mQkE1DnDZfrNbAgruQk", + "mint_authority": "565tK79EKhQSYybx52awrrFhD4m2ty198rrUBmiqq7xo", + "freeze_authority": "565tK79EKhQSYybx52awrrFhD4m2ty198rrUBmiqq7xo" + } + } + ] +} From 57056166ab07f43752b386529dd206cb7a64655f Mon Sep 17 00:00:00 2001 From: n00m4d Date: Fri, 31 Jan 2025 02:38:17 +0100 Subject: [PATCH 5/5] feat: optimize sql query and add more indexes --- ..._tests__get_different_assets_by_owner.snap | 2 +- ...ent_assets_by_owner_show_unverif_coll.snap | 2 +- ...earch_by_owner_with_show_zero_balance.snap | 10 ++-- migrations/12_fungible_and_assets_indexes.sql | 3 ++ postgre-client/src/asset_filter_client.rs | 54 +++++++++++++++---- 5 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 migrations/12_fungible_and_assets_indexes.sql diff --git a/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner.snap b/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner.snap index 0a1cb86d..5fd38834 100644 --- a/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner.snap +++ b/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner.snap @@ -1,6 +1,6 @@ --- source: integration_tests/src/general_scenario_tests.rs -assertion_line: 101 +assertion_line: 83 expression: response snapshot_kind: text --- diff --git a/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner_show_unverif_coll.snap b/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner_show_unverif_coll.snap index 0e05aca8..8569b6ff 100644 --- a/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner_show_unverif_coll.snap +++ b/integration_tests/src/snapshots/integration_tests__general_scenario_tests__get_different_assets_by_owner_show_unverif_coll.snap @@ -1,6 +1,6 @@ --- source: integration_tests/src/general_scenario_tests.rs -assertion_line: 124 +assertion_line: 102 expression: response snapshot_kind: text --- diff --git a/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap b/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap index 87658896..3b1f03e7 100644 --- a/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap +++ b/integration_tests/src/snapshots/integration_tests__regular_nft_tests__search_by_owner_with_show_zero_balance.snap @@ -1,6 +1,6 @@ --- source: integration_tests/src/regular_nft_tests.rs -assertion_line: 214 +assertion_line: 250 expression: response snapshot_kind: text --- @@ -54,10 +54,14 @@ snapshot_kind: text "frozen": false, "delegated": false, "delegate": null, - "ownership_model": "single", + "ownership_model": "token", "owner": "3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM" }, - "supply": null, + "supply": { + "print_max_supply": 0, + "print_current_supply": 0, + "edition_nonce": 252 + }, "mutable": true, "burnt": false, "lamports": 5616720, diff --git a/migrations/12_fungible_and_assets_indexes.sql b/migrations/12_fungible_and_assets_indexes.sql new file mode 100644 index 00000000..a55da181 --- /dev/null +++ b/migrations/12_fungible_and_assets_indexes.sql @@ -0,0 +1,3 @@ +CREATE INDEX fungible_tokens_fbt_owner_fbt_asset_fbt_balance_idx ON fungible_tokens (fbt_owner, fbt_asset, fbt_balance); + +CREATE INDEX assets_ast_owner_type_ast_pubkey_idx ON assets_v3 (ast_owner_type, ast_pubkey); \ No newline at end of file diff --git a/postgre-client/src/asset_filter_client.rs b/postgre-client/src/asset_filter_client.rs index beba52ce..a68271dc 100644 --- a/postgre-client/src/asset_filter_client.rs +++ b/postgre-client/src/asset_filter_client.rs @@ -104,7 +104,12 @@ impl PgClient { if group_clause_required { query_builder.push(" GROUP BY ast_pubkey, ast_slot_created, ast_slot_updated "); } - query_builder.push(") "); + + // Add ORDER BY clause + let direction = match (&order.sort_direction, order_reversed) { + (AssetSortDirection::Asc, true) | (AssetSortDirection::Desc, false) => " DESC ", + (AssetSortDirection::Asc, false) | (AssetSortDirection::Desc, true) => " ASC ", + }; // the function with a side-effect that mutates the query_builder if let Some(owner_address) = &filter.owner_address { @@ -113,9 +118,29 @@ impl PgClient { TokenType::Fungible | TokenType::NonFungible | TokenType::RegularNFT - | TokenType::CompressedNFT => {}, + | TokenType::CompressedNFT => { + query_builder.push(")"); + }, TokenType::All => { - query_builder.push(" UNION "); + // For this type of query we do union and apply limit with ordering for both parts of a query. + // It allows us to speed up the query for queries with wallets which has lots of assets. + // Not the cleanest approach. + // TODO: this may be improved + query_builder.push(" ORDER BY "); + query_builder.push(order.sort_by.to_string().replace("ast_", "")); + query_builder.push(direction); + + query_builder.push(" LIMIT "); + if let Some(page_num) = page.filter(|&p| p > 1) { + let lim = (page_num - 1) * limit; + query_builder.push_bind(lim as i64); + } else { + query_builder.push_bind(limit as i64); + } + + query_builder.push(")"); + + query_builder.push(" UNION ALL "); query_builder.push(" ( "); query_builder.push( "SELECT @@ -137,18 +162,29 @@ impl PgClient { " AND assets_v3.ast_is_collection_verified IS DISTINCT FROM FALSE", ); } + + query_builder.push(" ORDER BY "); + query_builder.push(order.sort_by.to_string()); + query_builder.push(direction); + + query_builder.push(" LIMIT "); + if let Some(page_num) = page.filter(|&p| p > 1) { + let lim = (page_num - 1) * limit; + query_builder.push_bind(lim as i64); + } else { + query_builder.push_bind(limit as i64); + } + query_builder.push(")"); }, } + } else { + query_builder.push(")"); } + } else { + query_builder.push(")"); } - // Add ORDER BY clause - let direction = match (&order.sort_direction, order_reversed) { - (AssetSortDirection::Asc, true) | (AssetSortDirection::Desc, false) => " DESC ", - (AssetSortDirection::Asc, false) | (AssetSortDirection::Desc, true) => " ASC ", - }; - query_builder.push(" ORDER BY "); query_builder.push(order.sort_by.to_string().replace("ast_", "")); query_builder.push(direction);