From 2e85b7375b2b08a94e69686e9415fa18239be77b Mon Sep 17 00:00:00 2001 From: Chii Yuen Date: Fri, 10 Jul 2026 05:01:44 +0800 Subject: [PATCH 1/3] feat: add support for getTransfersByAddress method --- CONTRIBUTING.md | 5 +- README.md | 5 +- Tiltfile | 7 +- crates/superbank-rpc/README.md | 18 +- crates/superbank-rpc/src/clickhouse/client.rs | 96 +++ crates/superbank-rpc/src/clickhouse/mod.rs | 6 +- .../superbank-rpc/src/clickhouse/queries.rs | 18 + .../superbank-rpc/src/clickhouse/sharding.rs | 1 + .../superbank-rpc/src/clickhouse/transfers.rs | 433 ++++++++++ crates/superbank-rpc/src/clickhouse/types.rs | 98 +++ crates/superbank-rpc/src/config.rs | 4 + crates/superbank-rpc/src/handlers/mod.rs | 4 + .../src/handlers/transactions.rs | 762 +++++++++++++++++- crates/superbank-rpc/src/handlers/types.rs | 75 ++ crates/superbank-rpc/src/server.rs | 1 + ddl/README.md | 5 +- ddl/cluster/transfers.sql | 651 +++++++++++++++ ddl/local/transfers.sql | 631 +++++++++++++++ ddl/replicated/transfers.sql | 651 +++++++++++++++ deploy/k8s/11-clickhouse-ddl-job.yaml | 1 + scripts/ci/release-e2e.sh | 1 + scripts/dev/run-local-rpc.sh | 7 + scripts/test/run-k6.sh | 13 + tests/k6/README.md | 34 +- tests/k6/lib/config.js | 68 ++ tests/k6/lib/rpc.js | 20 + ...k-rpc-validate-get-transfers-by-address.js | 242 ++++++ 27 files changed, 3840 insertions(+), 17 deletions(-) create mode 100644 crates/superbank-rpc/src/clickhouse/transfers.rs create mode 100644 ddl/cluster/transfers.sql create mode 100644 ddl/local/transfers.sql create mode 100644 ddl/replicated/transfers.sql create mode 100644 tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6ce4b0..0c3bb29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -176,12 +176,13 @@ Validation scenarios compare Superbank RPC responses against a reference endpoin REFERENCE_RPC_URL=https://api.mainnet-beta.solana.com scripts/test/run-k6.sh ``` -`getTransactionsForAddress` is a Superbank-specific method, so its validation scenario uses -`TFA_REFERENCE_RPC_URL` instead of the standard Solana reference endpoint: +`getTransactionsForAddress` and `getTransfersByAddress` are Superbank-specific methods, so its validation scenarios use +`TFA_REFERENCE_RPC_URL` and `TBA_REFERENCE_RPC_URL` instead of the standard Solana reference endpoint: ```bash REFERENCE_RPC_URL=https://api.mainnet-beta.solana.com \ TFA_REFERENCE_RPC_URL=http://localhost:8898 \ +TBA_REFERENCE_RPC_URL=http://localhost:8898 \ scripts/test/run-k6.sh ``` diff --git a/README.md b/README.md index 17b2b09..daf348d 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ ClickHouse. For single-node ClickHouse (local dev), apply the schemas under `ddl/local/` in this order. `transactions.sql` must be applied before the materialized-view schemas (`gsfa*.sql`, -`signatures.sql`, and `token_owner_activity.sql`) because those views read from the transactions -table. +`signatures.sql`, `token_owner_activity.sql`, and `transfers.sql`) because those views +read from the transactions table. ```bash cat ddl/local/transactions.sql | docker exec -i clickhouse clickhouse-client --multiquery @@ -93,6 +93,7 @@ cat ddl/local/gsfa.sql | docker exec -i clickhouse clickhouse-client --multiquer cat ddl/local/signatures.sql | docker exec -i clickhouse clickhouse-client --multiquery # Optional: required only for `tokenAccounts` filters in `getTransactionsForAddress`. cat ddl/local/token_owner_activity.sql | docker exec -i clickhouse clickhouse-client --multiquery +cat ddl/local/transfers.sql | docker exec -i clickhouse clickhouse-client --multiquery ``` If you use `gsfa_hot.sql` and want hot addresses excluded from the main GSFA table, apply diff --git a/Tiltfile b/Tiltfile index 8ac2921..ed0d948 100644 --- a/Tiltfile +++ b/Tiltfile @@ -117,6 +117,8 @@ data: {signatures} token_owner_activity.sql: | {token_owner_activity} + transfers.sql: | +{transfers} """.format( namespace=namespace, transactions=_indent_block(read_file("ddl/local/transactions.sql"), 4), @@ -126,6 +128,7 @@ data: gsfa_hot=_indent_block(read_file("ddl/local/gsfa_hot.sql"), 4), signatures=_indent_block(read_file("ddl/local/signatures.sql"), 4), token_owner_activity=_indent_block(read_file("ddl/local/token_owner_activity.sql"), 4), + transfers=_indent_block(read_file("ddl/local/transfers.sql"), 4), ) ) @@ -297,7 +300,8 @@ for f in \\ ddl/local/gsfa.sql \\ ddl/local/gsfa_hot.sql \\ ddl/local/signatures.sql \\ - ddl/local/token_owner_activity.sql + ddl/local/token_owner_activity.sql \\ + ddl/local/transfers.sql do echo "[apply-clickhouse-schema] Applying $f..." cat "$f" | kubectl -n "$ns" exec -i "$pod" -c clickhouse -- clickhouse-client --user "$ch_user" --password "$ch_password" --multiquery @@ -313,6 +317,7 @@ echo "[apply-clickhouse-schema] Done." "ddl/local/gsfa_hot.sql", "ddl/local/signatures.sql", "ddl/local/token_owner_activity.sql", + "ddl/local/transfers.sql", ], resource_deps=["clickhouse"], trigger_mode=TRIGGER_MODE_MANUAL, diff --git a/crates/superbank-rpc/README.md b/crates/superbank-rpc/README.md index debdfe5..943552f 100644 --- a/crates/superbank-rpc/README.md +++ b/crates/superbank-rpc/README.md @@ -23,6 +23,7 @@ writer that matches the same schemas). - `getInflationReward` - `getEpochSchedule` - `getTransactionsForAddress` (custom) +- `getTransfersByAddress` (custom) Notes: - JSON-RPC batch envelopes are supported. Batch execution is bounded by @@ -64,6 +65,13 @@ Notes: these aliases cannot be combined with same-side slot filters (`lt`/`lte` for `beforeSlot`, `gt`/`gte` for `untilSlot`). Token account filters require the token-owner activity table (see below). +- `getTransfersByAddress` supports historical SOL/SPL transfers with + top-level `with`, `direction=in|out`, `mint`, `solMode=merged|separate`, `sortOrder=asc|desc`, + `limit`, `paginationToken` (`slot:transactionIdx:instructionIdx:innerInstructionIdx:type`), + `commitment`, `minContextSlot`, and `filters.amount|slot|blockTime`. Amounts are raw integer + strings (lamports for SOL, token base units for SPL), with `uiAmount` derived from decimals. + The index is derived from pre/post balance deltas, so the opposite user account is nullable when + no counterparty can be inferred. This method requires the transfers table (see below). ## ClickHouse schemas @@ -81,10 +89,12 @@ Required files in the chosen set: Optional: - `gsfa_hot.sql` when using hot-address routing. - `token_owner_activity.sql` when using token-owner filters in `getTransactionsForAddress`. +- `transfers.sql` when using `getTransfersByAddress`. Apply `transactions.sql` before the materialized-view schemas (`gsfa*.sql`, `signatures.sql`, and -`token_owner_activity.sql`) because those views read from the transactions table. If you use -`gsfa_hot.sql`, apply `gsfa_nohot.sql` instead of `gsfa.sql`, then apply `gsfa_hot.sql`. +`token_owner_activity.sql`, `transfers.sql`) because those views read from the +transactions table. If you use `gsfa_hot.sql`, apply `gsfa_nohot.sql` instead of `gsfa.sql`, then +apply `gsfa_hot.sql`. ## Run @@ -377,6 +387,7 @@ CLI flags and environment variables (see `crates/superbank-rpc/src/config.rs`): | `--clickhouse-gsfa-hot-local-table` | `CLICKHOUSE_GSFA_HOT_LOCAL_TABLE` | `default.gsfa_hot_local` | Shard-local backing table behind `CLICKHOUSE_GSFA_HOT_TABLE`. | | `--clickhouse-signatures-local-table` | `CLICKHOUSE_SIGNATURES_LOCAL_TABLE` | — | — | | `--clickhouse-token-owner-activity-local-table` | `CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE` | — | — | +| `--clickhouse-transfers-local-table` | `CLICKHOUSE_TRANSFERS_LOCAL_TABLE` | — | — | | `--clickhouse-transactions-local-table` | `CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE` | — | — | | `--clickhouse-blocks-metadata-local-table` | `CLICKHOUSE_BLOCKS_METADATA_LOCAL_TABLE` | — | — | | `--clickhouse-shard-http-port` | `CLICKHOUSE_SHARD_HTTP_PORT` | — | — | @@ -392,6 +403,8 @@ Table selection (environment variables, read at startup): | `CLICKHOUSE_GSFA_HOT_TABLE` | `default.gsfa_hot` | — | | `CLICKHOUSE_SIGNATURE_STATUSES_TABLE` | `default.signatures` | — | | `CLICKHOUSE_TOKEN_OWNER_ACTIVITY_TABLE` | `default.token_owner_activity` | — | +| `CLICKHOUSE_TRANSFERS_TABLE` | `default.transfers` | Transfers ledger used by `getTransfersByAddress`. | +| `CLICKHOUSE_TRANSFERS_BY_ADDRESS_TABLE` | — | Legacy alias for `CLICKHOUSE_TRANSFERS_TABLE`. | Shard routing: When `CLICKHOUSE_SCOPE=shard-direct`, superbank-rpc discovers shards from `system.clusters` and @@ -462,6 +475,7 @@ Additional env flags: - Scope: `superbank-rpc` applies `use_query_condition_cache=1` only on selected historical address-filtered reads: - `getTransactionsForAddress` + - `getTransfersByAddress` - the transactions-table fallback path for `getSignaturesForAddress` - Point lookups and slot-range reads do not opt in. - This setting is enabled separately from the query-result cache via: diff --git a/crates/superbank-rpc/src/clickhouse/client.rs b/crates/superbank-rpc/src/clickhouse/client.rs index 4da471d..49a4dc3 100644 --- a/crates/superbank-rpc/src/clickhouse/client.rs +++ b/crates/superbank-rpc/src/clickhouse/client.rs @@ -27,6 +27,7 @@ use super::constants::DEFAULT_BUCKET_MODULUS; use super::gsfa::GsfaShardRouter; use super::queries::{ GSFA_REQUIRED_COLUMNS, SIGNATURES_REQUIRED_COLUMNS, TOKEN_OWNER_REQUIRED_COLUMNS, + TRANSFERS_REQUIRED_COLUMNS, }; use super::sharding::{ BucketColumn, ClusterRow, DescribeTableRow, RoutingPolicy, RoutingScope, RoutingTransport, @@ -416,11 +417,14 @@ pub struct ClickHouseClient { pub(crate) gsfa_hot_pubkeys: HashSet, pub(crate) signature_statuses_table: String, pub(crate) token_owner_activity_table: String, + pub(crate) transfers_table: String, pub(crate) signatures_local_table: Option, pub(crate) token_owner_activity_local_table: Option, + pub(crate) transfers_local_table: Option, pub(crate) transactions_local_table: Option, pub(crate) blocks_metadata_local_table: Option, pub(crate) token_owner_activity_available: bool, + pub(crate) transfers_available: bool, pub(crate) bucket_moduli: BucketModuli, pub(crate) allow_query_settings: bool, pub(crate) query_cache: QueryCacheConfig, @@ -630,6 +634,9 @@ impl ClickHouseClient { .unwrap_or_else(|_| "default.signatures".to_string()); let token_owner_activity_table = std::env::var("CLICKHOUSE_TOKEN_OWNER_ACTIVITY_TABLE") .unwrap_or_else(|_| "default.token_owner_activity".to_string()); + let transfers_table = std::env::var("CLICKHOUSE_TRANSFERS_TABLE") + .or_else(|_| std::env::var("CLICKHOUSE_TRANSFERS_BY_ADDRESS_TABLE")) + .unwrap_or_else(|_| "default.transfers".to_string()); let signatures_local_table = shard_routing .as_ref() @@ -639,6 +646,10 @@ impl ClickHouseClient { .as_ref() .and_then(|config| config.token_owner_activity_local_table.clone()) .or_else(|| derive_local_table_name(&token_owner_activity_table, None)); + let transfers_local_table = shard_routing + .as_ref() + .and_then(|config| config.transfers_local_table.clone()) + .or_else(|| derive_local_table_name(&transfers_table, None)); let transactions_local_table = shard_routing .as_ref() .and_then(|config| config.transactions_local_table.clone()) @@ -663,6 +674,9 @@ impl ClickHouseClient { if config.token_owner_activity_local_table.is_none() { config.token_owner_activity_local_table = token_owner_activity_local_table.clone(); } + if config.transfers_local_table.is_none() { + config.transfers_local_table = transfers_local_table.clone(); + } if config.transactions_local_table.is_none() { config.transactions_local_table = transactions_local_table.clone(); } @@ -688,11 +702,14 @@ impl ClickHouseClient { gsfa_hot_pubkeys: HashSet::new(), signature_statuses_table, token_owner_activity_table, + transfers_table, signatures_local_table, token_owner_activity_local_table, + transfers_local_table, transactions_local_table, blocks_metadata_local_table, token_owner_activity_available: true, + transfers_available: true, bucket_moduli: BucketModuli::default(), allow_query_settings: !std::env::var("CLICKHOUSE_DISABLE_QUERY_SETTINGS") .map(|value| env_truthy(&value)) @@ -1326,6 +1343,68 @@ impl ClickHouseClient { } } + let transfers_table = &self.transfers_table; + match self.startup_table_check { + ClickHouseStartupTableCheck::Count => { + match self + .with_timeout("startup_transfers_count", async { + self.client + .query(&format!("SELECT COUNT(*) FROM {}", transfers_table)) + .fetch_one::() + .await + .map_err(|e| ProcessingError::database(e.to_string(), e)) + }) + .await + { + Ok(count) => { + self.transfers_available = true; + tracing::info!( + "📊 Database initialized - {} table: {} rows", + transfers_table, + count + ); + } + Err(e) => { + self.transfers_available = false; + tracing::warn!( + "Transfers-by-address table '{}' unavailable; getTransfersByAddress disabled. Error: {}", + transfers_table, + e + ); + } + } + } + ClickHouseStartupTableCheck::Exists => { + match self + .with_timeout("startup_transfers_exists", async { + self.client + .query(&format!("SELECT count() FROM {} WHERE 0", transfers_table)) + .fetch_one::() + .await + .map(|_| ()) + .map_err(|e| ProcessingError::database(e.to_string(), e)) + }) + .await + { + Ok(()) => { + self.transfers_available = true; + tracing::info!( + "📊 Database initialized - {} table accessible", + transfers_table + ); + } + Err(e) => { + self.transfers_available = false; + tracing::warn!( + "Transfers-by-address table '{}' unavailable; getTransfersByAddress disabled. Error: {}", + transfers_table, + e + ); + } + } + } + } + let blocks_metadata_table = &self.blocks_metadata_table; match self.startup_table_check { ClickHouseStartupTableCheck::Count => { @@ -1517,6 +1596,22 @@ impl ClickHouseClient { ); self.token_owner_activity_local_table = None; } + + if let Some(local_table) = config.transfers_local_table.clone() + && let Err(e) = validate_table_schema_on_shards( + topology.as_ref(), + &local_table, + &TRANSFERS_REQUIRED_COLUMNS, + self.query_timeout, + ) + .await + { + tracing::warn!( + "Transfers shard routing disabled; local table validation failed: {}", + e + ); + self.transfers_local_table = None; + } } if hot_routing_configured { @@ -2132,6 +2227,7 @@ nodes: gsfa_local_table: None, signatures_local_table: None, token_owner_activity_local_table: None, + transfers_local_table: None, transactions_local_table: None, blocks_metadata_local_table: None, }; diff --git a/crates/superbank-rpc/src/clickhouse/mod.rs b/crates/superbank-rpc/src/clickhouse/mod.rs index bd9229b..80611d5 100644 --- a/crates/superbank-rpc/src/clickhouse/mod.rs +++ b/crates/superbank-rpc/src/clickhouse/mod.rs @@ -13,6 +13,7 @@ mod rows; mod sharding; mod signatures; mod transactions; +mod transfers; mod types; mod util; @@ -21,9 +22,10 @@ pub use client::{ClickHouseClient, ClickHouseClientOptions}; pub use types::TransactionsForAddressRecord; pub use types::{ BlockMetadataRecord, NumericFilter, PaginationToken, QueryTimings, SignatureFilter, - SignatureRecord, SignatureStatusRecord, SortOrder, StoredAccountsTransactionRecord, + SignatureRecord, SignatureStatusRecord, SolMode, SortOrder, StoredAccountsTransactionRecord, StoredBlockPayload, StoredBlockRecord, StoredTransactionRecord, TokenAccountsFilter, - TransactionStatusFilter, TransactionsForAddressQuery, + TokenTransferTypes, TransactionStatusFilter, TransactionsForAddressQuery, + TransferDirectionFilter, TransferPositionFilter, TransferRecord, TransfersByAddressQuery, }; pub(crate) use types::{ResolvedSignatureFilter, SignatureSlot, SlotBoundary}; diff --git a/crates/superbank-rpc/src/clickhouse/queries.rs b/crates/superbank-rpc/src/clickhouse/queries.rs index 80ff764..b10a861 100644 --- a/crates/superbank-rpc/src/clickhouse/queries.rs +++ b/crates/superbank-rpc/src/clickhouse/queries.rs @@ -235,6 +235,24 @@ pub(crate) const TOKEN_OWNER_REQUIRED_COLUMNS: [&str; 9] = [ "balance_changed", ]; +pub(crate) const TRANSFERS_REQUIRED_COLUMNS: [&str; 15] = [ + "signature", + "slot", + "slot_idx", + "transfer_idx", + "inner_instruction_idx", + "block_time", + "transfer_type", + "amount", + "mint", + "decimals", + "from_user_account", + "to_user_account", + "from_token_account", + "to_token_account", + "fee_amount", +]; + pub(crate) struct TransactionsForAddressTables<'a> { pub(crate) gsfa_table: &'a str, pub(crate) gsfa_bucket_modulus: u64, diff --git a/crates/superbank-rpc/src/clickhouse/sharding.rs b/crates/superbank-rpc/src/clickhouse/sharding.rs index 628e52c..2f89d64 100644 --- a/crates/superbank-rpc/src/clickhouse/sharding.rs +++ b/crates/superbank-rpc/src/clickhouse/sharding.rs @@ -48,6 +48,7 @@ pub(crate) struct ShardRoutingConfig { pub(crate) gsfa_local_table: Option, pub(crate) signatures_local_table: Option, pub(crate) token_owner_activity_local_table: Option, + pub(crate) transfers_local_table: Option, pub(crate) transactions_local_table: Option, pub(crate) blocks_metadata_local_table: Option, } diff --git a/crates/superbank-rpc/src/clickhouse/transfers.rs b/crates/superbank-rpc/src/clickhouse/transfers.rs new file mode 100644 index 0000000..8d63575 --- /dev/null +++ b/crates/superbank-rpc/src/clickhouse/transfers.rs @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +use std::str::FromStr; +use std::time::Instant; + +use serde::Deserialize; +use solana_sdk::pubkey::Pubkey; + +use crate::processing::{ProcessingError, ProcessingResult}; + +use super::QueryFreshnessClass; +use super::client::ClickHouseClient; +use super::types::{ + NumericFilter, QueryTimings, SolMode, SortOrder, TransferDirectionFilter, TransferRecord, + TransfersByAddressQuery, +}; +use super::util::pubkey_literal; + +const NATIVE_SOL_MINT: &str = "So11111111111111111111111111111111111111111"; +const WSOL_MINT: &str = "So11111111111111111111111111111111111111112"; + +#[derive(Deserialize, clickhouse::Row)] +struct TransferQueryRow { + signature: String, + slot: u64, + slot_idx: u32, + transfer_idx: u32, + inner_instruction_idx: u32, + block_time: Option, + transfer_type: String, + amount: String, + mint: Option, + decimals: Option, + from_user_account: Option, + to_user_account: Option, + from_token_account: Option, + to_token_account: Option, +} + +fn map_transfer_row(row: TransferQueryRow) -> TransferRecord { + TransferRecord { + signature: row.signature, + slot: row.slot, + slot_idx: row.slot_idx, + transfer_idx: row.transfer_idx, + inner_instruction_idx: row.inner_instruction_idx, + block_time: row.block_time, + transfer_type: row.transfer_type, + amount: row.amount, + mint: row.mint, + decimals: row.decimals, + from_user_account: row.from_user_account, + to_user_account: row.to_user_account, + from_token_account: row.from_token_account, + to_token_account: row.to_token_account, + } +} + +fn append_numeric_filter_conditions( + column: &str, + filter: &NumericFilter, + conditions: &mut Vec, +) { + if let Some(value) = &filter.eq { + conditions.push(format!("{column} = {value}")); + } + if let Some(value) = &filter.gte { + conditions.push(format!("{column} >= {value}")); + } + if let Some(value) = &filter.gt { + conditions.push(format!("{column} > {value}")); + } + if let Some(value) = &filter.lte { + conditions.push(format!("{column} <= {value}")); + } + if let Some(value) = &filter.lt { + conditions.push(format!("{column} < {value}")); + } +} + +fn append_pubkey_filter( + column: &str, + value: &str, + conditions: &mut Vec, +) -> ProcessingResult<()> { + let pubkey = Pubkey::from_str(value) + .map_err(|e| ProcessingError::deserialization("Invalid pubkey filter", e))?; + conditions.push(format!("{column} = {}", pubkey_literal(&pubkey))); + Ok(()) +} + +fn pubkey_literal_from_str(value: &str) -> ProcessingResult { + let pubkey = Pubkey::from_str(value) + .map_err(|e| ProcessingError::deserialization("Invalid pubkey filter", e))?; + Ok(pubkey_literal(&pubkey)) +} + +fn append_pagination_condition(query: &TransfersByAddressQuery, conditions: &mut Vec) { + let Some(token) = &query.pagination else { + return; + }; + let op = match query.sort_order { + SortOrder::Desc => "<", + SortOrder::Asc => ">", + }; + conditions.push(format!( + "(slot, slot_idx, transfer_idx, inner_instruction_idx, transfer_type) {op} ({}, {}, {}, {}, '{}')", + token.slot, + token.slot_idx, + token.transfer_idx, + token.inner_instruction_idx, + <&'static str>::from(token.transfer_type) + )); +} + +fn transfer_select_list(table_alias: &str) -> String { + format!( + "base58Encode({table_alias}.signature) AS signature, + {table_alias}.slot, + {table_alias}.slot_idx, + {table_alias}.transfer_idx, + {table_alias}.inner_instruction_idx, + {table_alias}.block_time, + {table_alias}.transfer_type, + {table_alias}.amount, + if(isNull({table_alias}.mint), NULL, base58Encode(assumeNotNull({table_alias}.mint))) AS mint, + {table_alias}.decimals, + if(isNull({table_alias}.from_user_account), NULL, base58Encode(assumeNotNull({table_alias}.from_user_account))) AS from_user_account, + if(isNull({table_alias}.to_user_account), NULL, base58Encode(assumeNotNull({table_alias}.to_user_account))) AS to_user_account, + if(isNull({table_alias}.from_token_account), NULL, base58Encode(assumeNotNull({table_alias}.from_token_account))) AS from_token_account, + if(isNull({table_alias}.to_token_account), NULL, base58Encode(assumeNotNull({table_alias}.to_token_account))) AS to_token_account" + ) +} + +fn append_shared_filter_conditions( + table_alias: &str, + query: &TransfersByAddressQuery, + conditions: &mut Vec, +) -> ProcessingResult<()> { + append_pagination_condition(query, conditions); + if let Some(filter) = &query.amount_filter { + append_numeric_filter_conditions( + &format!("toFloat64OrZero({table_alias}.amount)"), + filter, + conditions, + ); + } + if let Some(filter) = &query.slot_filter { + append_numeric_filter_conditions(&format!("{table_alias}.slot"), filter, conditions); + } + if let Some(filter) = &query.block_time_filter { + append_numeric_filter_conditions(&format!("{table_alias}.block_time"), filter, conditions); + } + if query.sol_mode == SolMode::Merged { + let native_sol_mint = pubkey_literal_from_str(NATIVE_SOL_MINT)?; + conditions.push(format!( + "({table_alias}.transfer_type != 'wrap' AND ({table_alias}.transfer_type != 'closeAccount' OR {table_alias}.mint = {native_sol_mint}))" + )); + } + if let Some(mint) = &query.mint { + if query.sol_mode == SolMode::Merged && (mint == NATIVE_SOL_MINT || mint == WSOL_MINT) { + conditions.push(format!( + "{table_alias}.mint IN ({}, {})", + pubkey_literal_from_str(NATIVE_SOL_MINT)?, + pubkey_literal_from_str(WSOL_MINT)? + )); + } else { + append_pubkey_filter(&format!("{table_alias}.mint"), mint, conditions)?; + } + } + Ok(()) +} + +fn build_transfer_branch( + table: &str, + table_alias: &str, + address_literal: &str, + user_column: &str, + counterparty_user_column: Option<&str>, + query: &TransfersByAddressQuery, +) -> ProcessingResult { + let mut conditions = vec!["1".to_string()]; + conditions.push(format!("{table_alias}.{user_column} = {address_literal}")); + if let (Some(counterparty_column), Some(with_account)) = + (counterparty_user_column, query.with_account.as_deref()) + { + append_pubkey_filter( + &format!("{table_alias}.{counterparty_column}"), + with_account, + &mut conditions, + )?; + } + append_shared_filter_conditions(table_alias, query, &mut conditions)?; + let where_clause = conditions.join(" AND "); + Ok(format!( + "SELECT + {select_list} + FROM {table} AS {table_alias} + WHERE {where_clause}", + select_list = transfer_select_list(table_alias), + table = table, + table_alias = table_alias, + where_clause = where_clause, + )) +} + +fn build_transfers_by_address_query( + table: &str, + query: &TransfersByAddressQuery, + settings_clause: &str, +) -> ProcessingResult { + let address = Pubkey::from_str(&query.address) + .map_err(|e| ProcessingError::deserialization("Invalid address", e))?; + let address_literal = pubkey_literal(&address); + + let from_branch = build_transfer_branch( + table, + "from_t", + &address_literal, + "from_user_account", + Some("to_user_account"), + query, + )?; + let to_branch = build_transfer_branch( + table, + "to_t", + &address_literal, + "to_user_account", + Some("from_user_account"), + query, + )?; + + let body = match query.direction { + TransferDirectionFilter::Out => from_branch, + TransferDirectionFilter::In => to_branch, + TransferDirectionFilter::Any => format!("{from_branch}\nUNION DISTINCT\n{to_branch}"), + }; + + let order = match query.sort_order { + SortOrder::Desc => "DESC", + SortOrder::Asc => "ASC", + }; + + Ok(format!( + "SELECT * + FROM ( + {body} + ) + ORDER BY slot {order}, slot_idx {order}, transfer_idx {order}, inner_instruction_idx {order}, transfer_type {order} + LIMIT {limit} + {settings_clause}", + body = body, + order = order, + limit = query.limit, + settings_clause = settings_clause, + )) +} + +impl ClickHouseClient { + pub async fn get_transfers_by_address( + &self, + query: &TransfersByAddressQuery, + ) -> ProcessingResult<(Vec, QueryTimings)> { + self.with_timeout("get_transfers_by_address", async { + if !self.transfers_available { + return Err(ProcessingError::database_msg(format!( + "getTransfersByAddress requires transfers table '{}'", + self.transfers_table + ))); + } + + let settings_clause = self.select_settings_clause_with_condition_cache( + "get_transfers_by_address", + QueryFreshnessClass::Historical, + ); + let query_sql = + build_transfers_by_address_query(&self.transfers_table, query, &settings_clause)?; + + let start = Instant::now(); + let mut cursor = self + .client + .query(&query_sql) + .fetch::() + .map_err(|e| ProcessingError::database(e.to_string(), e))?; + + let mut results = Vec::new(); + while let Some(row) = cursor + .next() + .await + .map_err(|e| ProcessingError::database(e.to_string(), e))? + { + results.push(row); + } + + let records = results + .into_iter() + .map(map_transfer_row) + .collect::>(); + let timings = QueryTimings { + elapsed_ms: start.elapsed().as_millis() as u64, + received_bytes: cursor.received_bytes(), + decoded_bytes: cursor.decoded_bytes(), + rows_read: Some(0), + rows_read_unknown: true, + rows_returned: records.len() as u64, + }; + + Ok((records, timings)) + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_query(sol_mode: SolMode) -> TransfersByAddressQuery { + TransfersByAddressQuery { + address: "11111111111111111111111111111111".to_string(), + limit: 100, + sort_order: SortOrder::Desc, + sol_mode, + pagination: None, + amount_filter: None, + slot_filter: None, + block_time_filter: None, + direction: TransferDirectionFilter::Any, + mint: None, + with_account: None, + } + } + + #[test] + fn with_account_filter_applies_to_both_union_branches() { + let mut query = base_query(SolMode::Merged); + query.with_account = Some("9vR6ssB1BdzhAgVEoQoeZdbsqknBA2PkFYDsjvbAS5jP".to_string()); + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains("from_t.to_user_account = ")); + assert!(sql.contains("to_t.from_user_account = ")); + assert!(sql.contains("UNION DISTINCT")); + assert!(sql.contains("FROM default.transfers AS from_t")); + assert!(sql.contains("FROM default.transfers AS to_t")); + } + + #[test] + fn out_direction_uses_from_branch_only() { + let mut query = base_query(SolMode::Merged); + query.direction = TransferDirectionFilter::Out; + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains("from_t.from_user_account = ")); + assert!(!sql.contains("UNION ALL")); + assert!(!sql.contains("to_t.")); + } + + #[test] + fn in_direction_uses_to_branch_only() { + let mut query = base_query(SolMode::Merged); + query.direction = TransferDirectionFilter::In; + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains("to_t.to_user_account = ")); + assert!(!sql.contains("UNION ALL")); + assert!(!sql.contains("from_t.")); + } + + #[test] + fn merged_sol_mode_excludes_wrap_and_wsol_close_account_only() { + let query = base_query(SolMode::Merged); + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains("transfer_type != 'wrap'")); + assert!(sql.contains("transfer_type != 'closeAccount' OR")); + assert!(sql.contains(&pubkey_literal_from_str(NATIVE_SOL_MINT).unwrap())); + assert!(!sql.contains("transfer_type NOT IN ('wrap', 'closeAccount')")); + } + + #[test] + fn separate_sol_mode_keeps_lifecycle_rows() { + let query = base_query(SolMode::Separate); + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(!sql.contains("transfer_type NOT IN ('wrap', 'closeAccount')")); + } + + #[test] + fn merged_sol_mode_matches_native_and_wrapped_sol_mints() { + let mut query = base_query(SolMode::Merged); + query.mint = Some(NATIVE_SOL_MINT.to_string()); + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains(".mint IN (")); + assert!(sql.contains(&pubkey_literal_from_str(NATIVE_SOL_MINT).unwrap())); + assert!(sql.contains(&pubkey_literal_from_str(WSOL_MINT).unwrap())); + } + + #[test] + fn pagination_condition_uses_helius_cursor_shape() { + let mut query = base_query(SolMode::Separate); + query.pagination = Some(super::super::TransferPositionFilter { + slot: 315073428, + slot_idx: 35, + transfer_idx: 1, + inner_instruction_idx: 0, + transfer_type: super::super::TokenTransferTypes::Transfer, + }); + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains( + "(slot, slot_idx, transfer_idx, inner_instruction_idx, transfer_type) < (315073428, 35, 1, 0, 'transfer')" + )); + assert!(sql.contains( + "ORDER BY slot DESC, slot_idx DESC, transfer_idx DESC, inner_instruction_idx DESC, transfer_type DESC" + )); + } +} diff --git a/crates/superbank-rpc/src/clickhouse/types.rs b/crates/superbank-rpc/src/clickhouse/types.rs index 02fe6da..b09706a 100644 --- a/crates/superbank-rpc/src/clickhouse/types.rs +++ b/crates/superbank-rpc/src/clickhouse/types.rs @@ -236,6 +236,104 @@ pub struct TransactionsForAddressRecord { pub block_time: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransferDirectionFilter { + Any, + In, + Out, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SolMode { + Merged, + Separate, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum TokenTransferTypes { + #[default] + Transfer, + Mint, + Burn, + Wrap, + CloseAccount, + ChangeOwner, + WithdrawWithheldFee, +} + +impl TryFrom<&str> for TokenTransferTypes { + type Error = String; + + fn try_from(value: &str) -> Result { + match value { + "transfer" => Ok(Self::Transfer), + "mint" => Ok(Self::Mint), + "burn" => Ok(Self::Burn), + "wrap" => Ok(Self::Wrap), + "closeAccount" => Ok(Self::CloseAccount), + "changeOwner" => Ok(Self::ChangeOwner), + "withdrawWithheldFee" => Ok(Self::WithdrawWithheldFee), + _ => Err(format!("Invalid transfer type: {value}")), + } + } +} + +impl From for &'static str { + fn from(value: TokenTransferTypes) -> Self { + match value { + TokenTransferTypes::Transfer => "transfer", + TokenTransferTypes::Mint => "mint", + TokenTransferTypes::Burn => "burn", + TokenTransferTypes::Wrap => "wrap", + TokenTransferTypes::CloseAccount => "closeAccount", + TokenTransferTypes::ChangeOwner => "changeOwner", + TokenTransferTypes::WithdrawWithheldFee => "withdrawWithheldFee", + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct TransferPositionFilter { + pub slot: u64, + pub slot_idx: u32, + pub transfer_idx: u32, + pub inner_instruction_idx: u32, + pub transfer_type: TokenTransferTypes, +} + +#[derive(Debug, Clone)] +pub struct TransfersByAddressQuery { + pub address: String, + pub limit: u64, + pub sort_order: SortOrder, + pub sol_mode: SolMode, + pub pagination: Option, + pub amount_filter: Option>, + pub slot_filter: Option>, + pub block_time_filter: Option>, + pub direction: TransferDirectionFilter, + pub mint: Option, + pub with_account: Option, +} + +#[derive(Debug, Clone)] +pub struct TransferRecord { + pub signature: String, + pub slot: u64, + pub slot_idx: u32, + pub transfer_idx: u32, + pub inner_instruction_idx: u32, + pub block_time: Option, + pub transfer_type: String, + pub amount: String, + pub mint: Option, + pub decimals: Option, + pub from_user_account: Option, + pub to_user_account: Option, + pub from_token_account: Option, + pub to_token_account: Option, +} + #[derive(Debug, Clone)] #[cfg_attr(feature = "disk-cache", derive(serde::Serialize, serde::Deserialize))] pub struct BlockMetadataRecord { diff --git a/crates/superbank-rpc/src/config.rs b/crates/superbank-rpc/src/config.rs index e2d7a2d..dbd1e8a 100644 --- a/crates/superbank-rpc/src/config.rs +++ b/crates/superbank-rpc/src/config.rs @@ -321,6 +321,10 @@ pub struct RpcConfig { #[arg(long, env = "CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE")] pub(crate) clickhouse_token_owner_activity_local_table: Option, + /// Local transfers table name on each shard (defaults to CLICKHOUSE_TRANSFERS_TABLE + _local). + #[arg(long, env = "CLICKHOUSE_TRANSFERS_LOCAL_TABLE")] + pub(crate) clickhouse_transfers_local_table: Option, + /// Local transactions table name on each shard (defaults to CLICKHOUSE_TRANSACTION_TABLE + _local). #[arg(long, env = "CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE")] pub(crate) clickhouse_transactions_local_table: Option, diff --git a/crates/superbank-rpc/src/handlers/mod.rs b/crates/superbank-rpc/src/handlers/mod.rs index d6989e6..0810cb8 100644 --- a/crates/superbank-rpc/src/handlers/mod.rs +++ b/crates/superbank-rpc/src/handlers/mod.rs @@ -531,6 +531,7 @@ fn metrics_method_label(method: &str) -> &'static str { match method { "getSignaturesForAddress" => "getSignaturesForAddress", "getTransactionsForAddress" => "getTransactionsForAddress", + "getTransfersByAddress" => "getTransfersByAddress", "getSignatureStatuses" => "getSignatureStatuses", "getBlock" => "getBlock", "getBlockHeight" => "getBlockHeight", @@ -855,6 +856,9 @@ async fn dispatch_json_rpc_request( transactions::handle_get_transactions_for_address(state, id_for_dispatch, params) .await } + "getTransfersByAddress" => { + transactions::handle_get_transfers_by_address(state, id_for_dispatch, params).await + } "getSignatureStatuses" => { signatures::handle_get_signature_statuses(state, id_for_dispatch, params).await } diff --git a/crates/superbank-rpc/src/handlers/transactions.rs b/crates/superbank-rpc/src/handlers/transactions.rs index ecf2852..64b8f89 100644 --- a/crates/superbank-rpc/src/handlers/transactions.rs +++ b/crates/superbank-rpc/src/handlers/transactions.rs @@ -22,15 +22,18 @@ use tracing::error; use crate::clickhouse::{ NumericFilter, PaginationToken, QueryTimings, ResolvedSignatureFilter, SignatureFilter, - SignatureSlot, SortOrder, StoredTransactionRecord, TokenAccountsFilter, - TransactionStatusFilter, TransactionsForAddressQuery, + SignatureSlot, SolMode, SortOrder, StoredTransactionRecord, TokenAccountsFilter, + TokenTransferTypes, TransactionStatusFilter, TransactionsForAddressQuery, + TransferDirectionFilter, TransferPositionFilter, TransferRecord, TransfersByAddressQuery, }; use crate::handlers::{ RouteMetric, types::{ GetTransactionsForAddressFilters, GetTransactionsForAddressOptions, - TransactionsForAddressDetails, TransactionsForAddressFullInfo, - TransactionsForAddressResult, TransactionsForAddressSignaturesInfo, reject_unknown_fields, + GetTransfersByAddressOptions, TransactionsForAddressDetails, + TransactionsForAddressFullInfo, TransactionsForAddressResult, + TransactionsForAddressSignaturesInfo, TransfersByAddressInfo, TransfersByAddressResult, + reject_unknown_fields, }, }; use crate::hydration::{TransactionHydrationError, hydrate_transaction_record}; @@ -57,6 +60,8 @@ const GET_TRANSACTION_INVALID_SIGNATURE_MESSAGE: &str = "Invalid params: signature is not a valid transaction signature"; const GET_TRANSACTION_DENYLISTED_SIGNATURES: [&str; 1] = ["1111111111111111111111111111111111111111111111111111111111111111"]; +const NATIVE_SOL_MINT: &str = "So11111111111111111111111111111111111111111"; +const WSOL_MINT: &str = "So11111111111111111111111111111111111111112"; #[derive(Debug)] struct ParsedGetTransactionConfig { @@ -242,6 +247,172 @@ async fn respond_with_hydrated_transaction( } } +fn parse_transfer_pagination_token(token: &str) -> Result { + let token = token.trim(); + if token.is_empty() { + return Err("Invalid params: paginationToken is empty"); + } + + let mut parts = token.split(':'); + let slot = parts + .next() + .ok_or( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + )? + .parse::() + .map_err(|_| "Invalid params: paginationToken slot is not a number")?; + let transaction_idx = parts + .next() + .ok_or( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + )? + .parse::() + .map_err(|_| "Invalid params: paginationToken transactionIdx is not a number")?; + let instruction_idx = parts + .next() + .ok_or( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + )? + .parse::() + .map_err(|_| "Invalid params: paginationToken instructionIdx is not a number")?; + let inner_instruction_idx = parts + .next() + .ok_or( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + )? + .parse::() + .map_err(|_| "Invalid params: paginationToken innerInstructionIdx is not a number")?; + let transfer_type = parts + .next() + .ok_or( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + )?; + if transfer_type.is_empty() || parts.next().is_some() { + return Err( + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type", + ); + } + let transfer_type = TokenTransferTypes::try_from(transfer_type).map_err(|_| { + "Invalid params: paginationToken type must be one of transfer, mint, burn, wrap, closeAccount, changeOwner, withdrawWithheldFee" + })?; + + Ok(TransferPositionFilter { + slot, + slot_idx: transaction_idx, + transfer_idx: instruction_idx, + inner_instruction_idx, + transfer_type, + }) +} + +fn parse_transfer_direction_filter( + value: Option, +) -> Result { + match value + .as_deref() + .unwrap_or("any") + .trim() + .to_ascii_lowercase() + .as_str() + { + "any" => Ok(TransferDirectionFilter::Any), + "in" => Ok(TransferDirectionFilter::In), + "out" => Ok(TransferDirectionFilter::Out), + other => Err(format!("Invalid params: unsupported direction '{other}'")), + } +} + +fn parse_sol_mode(value: Option<&str>) -> Result { + match value + .unwrap_or("merged") + .trim() + .to_ascii_lowercase() + .as_str() + { + "merged" => Ok(SolMode::Merged), + "separate" => Ok(SolMode::Separate), + other => Err(format!("Invalid params: unsupported solMode '{other}'")), + } +} + +fn map_transfer_record_to_response( + record: TransferRecord, + sol_mode: SolMode, + confirmation_status: &str, +) -> TransfersByAddressInfo { + let mut mint = record.mint.unwrap_or_else(|| NATIVE_SOL_MINT.to_string()); + if sol_mode == SolMode::Merged && mint == WSOL_MINT { + mint = NATIVE_SOL_MINT.to_string(); + } + let decimals = record + .decimals + .or_else(|| (mint == NATIVE_SOL_MINT).then_some(9)); + let ui_amount = format_ui_amount(&record.amount, decimals); + let (from_token_account, to_token_account) = if mint == NATIVE_SOL_MINT { + (None, None) + } else { + (record.from_token_account, record.to_token_account) + }; + + TransfersByAddressInfo { + signature: record.signature, + slot: record.slot, + block_time: record.block_time, + transfer_type: record.transfer_type, + from_user_account: record.from_user_account, + to_user_account: record.to_user_account, + from_token_account, + to_token_account, + mint, + amount: record.amount, + decimals, + ui_amount, + confirmation_status: confirmation_status.to_string(), + transaction_idx: record.slot_idx, + instruction_idx: record.transfer_idx, + inner_instruction_idx: record.inner_instruction_idx, + } +} + +fn format_ui_amount(amount: &str, decimals: Option) -> String { + let decimals = decimals.unwrap_or(0) as usize; + if decimals == 0 { + return amount.to_string(); + } + + let amount = amount.trim(); + let negative = amount.starts_with('-'); + let digits = amount.trim_start_matches('-').trim_start_matches('+'); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return amount.to_string(); + } + + let mut whole = if digits.len() > decimals { + digits[..digits.len() - decimals].to_string() + } else { + "0".to_string() + }; + let mut fraction = if digits.len() > decimals { + digits[digits.len() - decimals..].to_string() + } else { + format!("{:0>width$}", digits, width = decimals) + }; + while fraction.ends_with('0') { + fraction.pop(); + } + if fraction.is_empty() { + if negative && whole != "0" { + whole.insert(0, '-'); + } + whole + } else { + if negative && whole != "0" { + whole.insert(0, '-'); + } + format!("{whole}.{fraction}") + } +} + pub(crate) async fn handle_get_transaction( state: Arc, id: Value, @@ -1830,6 +2001,293 @@ pub(crate) async fn handle_get_transactions_for_address( Ok(resp) } +pub(crate) async fn handle_get_transfers_by_address( + state: Arc, + id: Value, + params: Option>, +) -> Result { + let mut route = RouteMetric::for_state("getTransfersByAddress", state.as_ref()); + + let Some(mut params) = params.filter(|v| !v.is_empty()) else { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + "Invalid params: missing address", + None, + )); + }; + + let address_value = params.remove(0); + let address = match address_value.as_str() { + Some(value) => value, + None => { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + "Invalid params: address must be a string", + None, + )); + } + }; + + if Pubkey::from_str(address).is_err() { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + "Invalid param: Invalid", + None, + )); + } + + let options = match params.into_iter().next() { + Some(options_value) => { + if options_value.is_null() { + GetTransfersByAddressOptions::default() + } else { + match serde_json::from_value::(options_value) { + Ok(parsed) => parsed, + Err(e) => { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + format!("Invalid params: failed to parse options ({e})"), + None, + )); + } + } + } + } + None => GetTransfersByAddressOptions::default(), + }; + + let sort_order = match options + .sort_order + .as_deref() + .unwrap_or("desc") + .to_lowercase() + .as_str() + { + "asc" => SortOrder::Asc, + "desc" => SortOrder::Desc, + other => { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + format!("Invalid params: unsupported sortOrder '{other}'"), + None, + )); + } + }; + + let max_limit = state.max_signatures_limit.min(100); + let requested_limit = options.limit.unwrap_or(100); + if requested_limit == 0 { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + format!("Invalid limit; max {}", max_limit), + None, + )); + } + let limit = requested_limit.min(max_limit); + + if let Some(commitment) = options.commitment.as_deref() { + match commitment.trim().to_ascii_lowercase().as_str() { + "finalized" | "confirmed" => {} + "processed" => { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + "Only confirmed or finalized commitments are supported", + Some(json!({ "requestedCommitment": commitment })), + )); + } + other => { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + format!("Invalid params: unsupported commitment '{other}'"), + None, + )); + } + } + } + + let sol_mode = match parse_sol_mode(options.sol_mode.as_deref()) { + Ok(value) => value, + Err(message) => { + route.invalid_params(); + return Ok(json_rpc_error_response(id, -32602, message, None)); + } + }; + + if let Some(min_context_slot) = options.min_context_slot { + let commitment = options + .commitment + .as_deref() + .unwrap_or("finalized") + .trim() + .to_ascii_lowercase(); + let min_commitment = match commitment.as_str() { + "confirmed" => CommitmentLevel::Confirmed, + _ => CommitmentLevel::Finalized, + }; + + let (context_slot, context_source) = match state + .resolve_latest_slot_with_source("get_transfers_by_address_min_context", min_commitment) + .await + { + Ok(result) => result, + Err(e) => { + metrics::backend_error("get_latest_finalized_slot"); + error!( + "Failed to fetch latest slot for getTransfersByAddress minContextSlot check: {}", + e + ); + return Ok(json_rpc_internal_error_response(id)); + } + }; + match context_source { + LatestSlotSource::ClickHouse => route.source_clickhouse(), + #[cfg(feature = "grpc-head-cache")] + LatestSlotSource::HeadCache => route.source_head_cache(), + } + + if context_slot < min_context_slot { + let code = JSON_RPC_SERVER_ERROR_MIN_CONTEXT_SLOT_NOT_REACHED as i32; + route.rpc_error(); + return Ok(json_rpc_error_response( + id, + code, + "Minimum context slot has not been reached", + Some(json!({ "contextSlot": context_slot })), + )); + } + } + + let pagination = match options.pagination_token.as_deref() { + Some(token) => match parse_transfer_pagination_token(token) { + Ok(token) => Some(token), + Err(message) => { + route.invalid_params(); + return Ok(json_rpc_error_response(id, -32602, message, None)); + } + }, + None => None, + }; + + let filters = options.filters.unwrap_or_default(); + let direction = match parse_transfer_direction_filter(options.direction) { + Ok(value) => value, + Err(message) => { + route.invalid_params(); + return Ok(json_rpc_error_response(id, -32602, message, None)); + } + }; + + for (name, value) in [ + ("mint", options.mint.as_deref()), + ("with", options.with_account.as_deref()), + ] { + if let Some(value) = value + && Pubkey::from_str(value).is_err() + { + route.invalid_params(); + return Ok(json_rpc_error_response( + id, + -32602, + format!("Invalid params: {name} must be a valid pubkey"), + None, + )); + } + } + + let amount_filter = filters.amount.map(|filter| NumericFilter { + gte: filter.gte, + gt: filter.gt, + lte: filter.lte, + lt: filter.lt, + eq: filter.eq, + }); + let slot_filter = filters.slot.map(|filter| NumericFilter { + gte: filter.gte, + gt: filter.gt, + lte: filter.lte, + lt: filter.lt, + eq: filter.eq, + }); + let block_time_filter = filters.block_time.map(|filter| NumericFilter { + gte: filter.gte, + gt: filter.gt, + lte: filter.lte, + lt: filter.lt, + eq: filter.eq, + }); + let query = TransfersByAddressQuery { + address: address.to_string(), + limit, + sort_order, + sol_mode, + pagination, + amount_filter, + slot_filter, + block_time_filter, + direction, + mint: options.mint.clone(), + with_account: options.with_account.clone(), + }; + + route.source_clickhouse(); + let (records, timings) = match state.clickhouse.get_transfers_by_address(&query).await { + Ok(result) => result, + Err(e) => { + metrics::backend_error("get_transfers_by_address"); + error!( + "Failed to query ClickHouse for getTransfersByAddress: {}", + e + ); + return Ok(json_rpc_internal_error_response(id)); + } + }; + + let pagination_token = records.last().map(|record| { + let transfer_type = + TokenTransferTypes::try_from(record.transfer_type.as_str()).unwrap_or_default(); + format!( + "{}:{}:{}:{}:{}", + record.slot, + record.slot_idx, + record.transfer_idx, + record.inner_instruction_idx, + <&'static str>::from(transfer_type) + ) + }); + let confirmation_status = options.commitment.as_deref().unwrap_or("finalized"); + let data = records + .into_iter() + .map(|record| map_transfer_record_to_response(record, sol_mode, confirmation_status)) + .collect::>(); + + route.success(); + let mut resp = json_rpc_success_response( + id, + TransfersByAddressResult { + data, + pagination_token, + }, + ); + add_downstream_header(&mut resp, &timings); + Ok(resp) +} + #[cfg(test)] mod tests { use super::*; @@ -1882,6 +2340,302 @@ mod tests { assert!(err.contains("unexpected")); } + #[test] + fn token_transfer_types_parse_supported_values() { + assert_eq!( + TokenTransferTypes::try_from("transfer"), + Ok(TokenTransferTypes::Transfer) + ); + assert_eq!( + TokenTransferTypes::try_from("changeOwner"), + Ok(TokenTransferTypes::ChangeOwner) + ); + assert_eq!( + TokenTransferTypes::try_from("withdrawWithheldFee"), + Ok(TokenTransferTypes::WithdrawWithheldFee) + ); + + assert!(TokenTransferTypes::try_from("unsupported").is_err()); + } + + #[test] + fn parse_transfer_pagination_token_accepts_helius_transfer_types() { + for transfer_type in [ + "transfer", + "mint", + "burn", + "wrap", + "closeAccount", + "changeOwner", + "withdrawWithheldFee", + ] { + let parsed = parse_transfer_pagination_token(&format!("10:2:3:0:{transfer_type}")) + .expect("pagination token should parse"); + + assert_eq!(parsed.slot, 10); + assert_eq!(parsed.slot_idx, 2); + assert_eq!(parsed.transfer_idx, 3); + assert_eq!(parsed.inner_instruction_idx, 0); + assert_eq!( + parsed.transfer_type, + TokenTransferTypes::try_from(transfer_type).expect("transfer type should parse") + ); + } + } + + #[test] + fn parse_transfer_pagination_token_rejects_unknown_transfer_type() { + let err = parse_transfer_pagination_token("10:2:3:0:splTransfer") + .expect_err("unsupported transfer type should fail"); + + assert_eq!( + err, + "Invalid params: paginationToken type must be one of transfer, mint, burn, wrap, closeAccount, changeOwner, withdrawWithheldFee" + ); + } + + #[test] + fn parse_transfer_pagination_token_rejects_empty_token() { + let err = + parse_transfer_pagination_token(" ").expect_err("empty pagination token should fail"); + + assert_eq!(err, "Invalid params: paginationToken is empty"); + } + + #[test] + fn parse_transfer_pagination_token_rejects_missing_or_extra_parts() { + let missing_inner_idx = parse_transfer_pagination_token("10:2:3:transfer") + .expect_err("missing inner instruction index should fail"); + let extra_part = parse_transfer_pagination_token("10:2:3:0:transfer:extra") + .expect_err("extra pagination token part should fail"); + + assert_eq!( + missing_inner_idx, + "Invalid params: paginationToken innerInstructionIdx is not a number" + ); + assert_eq!( + extra_part, + "Invalid params: paginationToken must be slot:transactionIdx:instructionIdx:innerInstructionIdx:type" + ); + } + + #[test] + fn parse_transfer_pagination_token_rejects_non_numeric_positions() { + let invalid_slot = parse_transfer_pagination_token("abc:2:3:0:transfer") + .expect_err("invalid slot should fail"); + let invalid_transaction_idx = parse_transfer_pagination_token("10:abc:3:0:transfer") + .expect_err("invalid transaction index should fail"); + let invalid_instruction_idx = parse_transfer_pagination_token("10:2:abc:0:transfer") + .expect_err("invalid instruction index should fail"); + let invalid_inner_idx = parse_transfer_pagination_token("10:2:3:abc:transfer") + .expect_err("invalid inner instruction index should fail"); + + assert_eq!( + invalid_slot, + "Invalid params: paginationToken slot is not a number" + ); + assert_eq!( + invalid_transaction_idx, + "Invalid params: paginationToken transactionIdx is not a number" + ); + assert_eq!( + invalid_instruction_idx, + "Invalid params: paginationToken instructionIdx is not a number" + ); + assert_eq!( + invalid_inner_idx, + "Invalid params: paginationToken innerInstructionIdx is not a number" + ); + } + + #[test] + fn parse_transfer_direction_filter_defaults_to_any_and_normalizes_values() { + assert_eq!( + parse_transfer_direction_filter(None).expect("default direction should parse"), + TransferDirectionFilter::Any + ); + assert_eq!( + parse_transfer_direction_filter(Some(" IN ".to_string())) + .expect("in direction should parse"), + TransferDirectionFilter::In + ); + assert_eq!( + parse_transfer_direction_filter(Some("out".to_string())) + .expect("out direction should parse"), + TransferDirectionFilter::Out + ); + } + + #[test] + fn parse_transfer_direction_filter_rejects_unknown_value() { + let err = parse_transfer_direction_filter(Some("sideways".to_string())) + .expect_err("unsupported direction should fail"); + + assert_eq!(err, "Invalid params: unsupported direction 'sideways'"); + } + + #[test] + fn parse_sol_mode_defaults_to_merged_and_accepts_supported_modes() { + assert_eq!( + parse_sol_mode(None).expect("default sol mode should parse"), + SolMode::Merged + ); + assert_eq!( + parse_sol_mode(Some(" merged ")).expect("merged sol mode should parse"), + SolMode::Merged + ); + assert_eq!( + parse_sol_mode(Some("SEPARATE")).expect("separate sol mode should parse"), + SolMode::Separate + ); + } + + #[test] + fn parse_sol_mode_rejects_unknown_value() { + let err = parse_sol_mode(Some("split")).expect_err("unsupported sol mode should fail"); + + assert_eq!(err, "Invalid params: unsupported solMode 'split'"); + } + + #[test] + fn map_transfer_record_to_response_classifies_mint_and_burn_accounts() { + let owner = "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC".to_string(); + let token_account = "6Tst7EUiwCnoiiJbess1mhiBbq4gPoBR4iu8pDb8fZf3".to_string(); + let mint_record = TransferRecord { + signature: "sig".to_string(), + slot: 1, + slot_idx: 2, + transfer_idx: 3, + inner_instruction_idx: 0, + block_time: Some(4), + transfer_type: "mint".to_string(), + amount: "100".to_string(), + mint: Some("EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v".to_string()), + decimals: Some(6), + from_user_account: None, + to_user_account: Some(owner.clone()), + from_token_account: None, + to_token_account: Some(token_account.clone()), + }; + let mapped = + map_transfer_record_to_response(mint_record.clone(), SolMode::Merged, "finalized"); + assert_eq!(mapped.transfer_type, "mint"); + assert_eq!(mapped.from_user_account, None); + assert_eq!(mapped.to_user_account.as_deref(), Some(owner.as_str())); + assert_eq!(mapped.from_token_account, None); + assert_eq!( + mapped.to_token_account.as_deref(), + Some(token_account.as_str()) + ); + + let burn = TransferRecord { + transfer_type: "burn".to_string(), + from_user_account: Some(owner.clone()), + to_user_account: None, + from_token_account: Some(token_account.clone()), + to_token_account: None, + ..mint_record + }; + let mapped = map_transfer_record_to_response(burn, SolMode::Merged, "finalized"); + assert_eq!(mapped.transfer_type, "burn"); + assert_eq!(mapped.from_user_account.as_deref(), Some(owner.as_str())); + assert_eq!(mapped.to_user_account, None); + assert_eq!( + mapped.from_token_account.as_deref(), + Some(token_account.as_str()) + ); + assert_eq!(mapped.to_token_account, None); + } + + #[test] + fn map_transfer_record_to_response_includes_counterparty_token_account_for_spl_transfers() { + let owner = "HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC".to_string(); + let counterparty = "9vR6ssB1BdzhAgVEoQoeZdbsqknBA2PkFYDsjvbAS5jP".to_string(); + let from_token = "CrExqHvA6Nws8ZRH7kGoTurbsb6Rm7RKFSKbVe6dA1m7".to_string(); + let to_token = "HcvK3EJ74iM9g11cUgsaPvLSrhCvCwcrWxBNd87LsC1x".to_string(); + let record = TransferRecord { + signature: "sig".to_string(), + slot: 1, + slot_idx: 2, + transfer_idx: 3, + inner_instruction_idx: 0, + block_time: Some(4), + transfer_type: "transfer".to_string(), + amount: "2500000".to_string(), + mint: Some("EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v".to_string()), + decimals: Some(6), + from_user_account: Some(owner.clone()), + to_user_account: Some(counterparty.clone()), + from_token_account: Some(from_token.clone()), + to_token_account: Some(to_token.clone()), + }; + let mapped = map_transfer_record_to_response(record, SolMode::Merged, "finalized"); + assert_eq!(mapped.transfer_type, "transfer"); + assert_eq!(mapped.from_user_account.as_deref(), Some(owner.as_str())); + assert_eq!( + mapped.to_user_account.as_deref(), + Some(counterparty.as_str()) + ); + assert_eq!( + mapped.from_token_account.as_deref(), + Some(from_token.as_str()) + ); + assert_eq!(mapped.to_token_account.as_deref(), Some(to_token.as_str())); + } + + #[test] + fn map_transfer_record_to_response_omits_token_accounts_for_native_sol() { + let record = TransferRecord { + signature: "sig".to_string(), + slot: 1, + slot_idx: 2, + transfer_idx: 3, + inner_instruction_idx: 0, + block_time: Some(4), + transfer_type: "transfer".to_string(), + amount: "1000".to_string(), + mint: Some(NATIVE_SOL_MINT.to_string()), + decimals: Some(9), + from_user_account: Some("9vR6ssB1BdzhAgVEoQoeZdbsqknBA2PkFYDsjvbAS5jP".to_string()), + to_user_account: Some("HLnpSz9h2S4hiLQ43rnSD9XkcUThA7B8hQMKmDaiTLcC".to_string()), + from_token_account: None, + to_token_account: None, + }; + let mapped = map_transfer_record_to_response(record, SolMode::Merged, "finalized"); + assert_eq!(mapped.from_token_account, None); + assert_eq!(mapped.to_token_account, None); + } + + #[test] + fn format_ui_amount_returns_raw_amount_without_decimals() { + assert_eq!(format_ui_amount("1234500", None), "1234500"); + assert_eq!(format_ui_amount("1234500", Some(0)), "1234500"); + } + + #[test] + fn format_ui_amount_places_decimal_point_and_trims_trailing_zeroes() { + assert_eq!(format_ui_amount("1234500", Some(6)), "1.2345"); + assert_eq!(format_ui_amount("1000000", Some(6)), "1"); + } + + #[test] + fn format_ui_amount_left_pads_fractional_amounts() { + assert_eq!(format_ui_amount("42", Some(6)), "0.000042"); + assert_eq!(format_ui_amount("1", Some(2)), "0.01"); + } + + #[test] + fn format_ui_amount_preserves_negative_sign_for_nonzero_whole_amounts() { + assert_eq!(format_ui_amount("-1234500", Some(6)), "-1.2345"); + assert_eq!(format_ui_amount("-1000000", Some(6)), "-1"); + } + + #[test] + fn format_ui_amount_returns_trimmed_raw_amount_for_invalid_digits() { + assert_eq!(format_ui_amount(" 12x ", Some(2)), "12x"); + assert_eq!(format_ui_amount("-", Some(2)), "-"); + } + #[test] fn get_transactions_for_address_slot_aliases_populate_slot_filter() { let mut filters = GetTransactionsForAddressFilters::default(); diff --git a/crates/superbank-rpc/src/handlers/types.rs b/crates/superbank-rpc/src/handlers/types.rs index 55be1c1..468a515 100644 --- a/crates/superbank-rpc/src/handlers/types.rs +++ b/crates/superbank-rpc/src/handlers/types.rs @@ -149,6 +149,46 @@ pub(crate) struct GetTransactionsForAddressOptions { pub(crate) filters: Option, } +#[derive(Debug, Deserialize, Default)] +pub(crate) struct GetTransfersByAddressFilters { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) amount: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) slot: Option>, + #[serde(rename = "blockTime")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) block_time: Option>, +} + +#[derive(Debug, Deserialize, Default)] +pub(crate) struct GetTransfersByAddressOptions { + #[serde(rename = "with")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) with_account: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) direction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) mint: Option, + #[serde(rename = "solMode")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) sol_mode: Option, + #[serde(rename = "sortOrder")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) sort_order: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) limit: Option, + #[serde(rename = "paginationToken")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pagination_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) commitment: Option, + #[serde(rename = "minContextSlot")] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) min_context_slot: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) filters: Option, +} + pub(crate) const MAX_GET_BLOCKS_RANGE: u64 = 500_000; #[derive(Debug, Deserialize)] @@ -203,6 +243,41 @@ pub(crate) struct TransactionsForAddressResult { pub(crate) pagination_token: Option, } +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TransfersByAddressInfo { + pub(crate) signature: String, + pub(crate) slot: u64, + #[serde(rename = "blockTime")] + pub(crate) block_time: Option, + #[serde(rename = "type")] + pub(crate) transfer_type: String, + pub(crate) from_user_account: Option, + pub(crate) to_user_account: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) from_token_account: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) to_token_account: Option, + pub(crate) mint: String, + pub(crate) amount: String, + pub(crate) decimals: Option, + pub(crate) ui_amount: String, + pub(crate) confirmation_status: String, + #[serde(rename = "transactionIdx")] + pub(crate) transaction_idx: u32, + #[serde(rename = "instructionIdx")] + pub(crate) instruction_idx: u32, + #[serde(rename = "innerInstructionIdx")] + pub(crate) inner_instruction_idx: u32, +} + +#[derive(Debug, Serialize)] +pub(crate) struct TransfersByAddressResult { + pub(crate) data: Vec, + #[serde(rename = "paginationToken")] + pub(crate) pagination_token: Option, +} + pub(crate) fn reject_unknown_fields(value: &Value, allowed: &[&str]) -> Result<(), String> { let Some(object) = value.as_object() else { return Ok(()); diff --git a/crates/superbank-rpc/src/server.rs b/crates/superbank-rpc/src/server.rs index 88937ef..96056c1 100644 --- a/crates/superbank-rpc/src/server.rs +++ b/crates/superbank-rpc/src/server.rs @@ -67,6 +67,7 @@ fn build_shard_routing_config(args: &RpcConfig) -> Option { token_owner_activity_local_table: args .clickhouse_token_owner_activity_local_table .clone(), + transfers_local_table: args.clickhouse_transfers_local_table.clone(), transactions_local_table: args.clickhouse_transactions_local_table.clone(), blocks_metadata_local_table: args.clickhouse_blocks_metadata_local_table.clone(), }) diff --git a/ddl/README.md b/ddl/README.md index ac141e1..019bc5d 100644 --- a/ddl/README.md +++ b/ddl/README.md @@ -16,11 +16,12 @@ Each folder contains the same file basenames: - `gsfa_hot.sql` - `signatures.sql` - `token_owner_activity.sql` +- `transfers.sql` Pick one folder and apply the matching schema set consistently. Apply `transactions.sql` before materialized-view files such as `gsfa*.sql`, `signatures.sql`, and -`token_owner_activity.sql`; those views select from the transactions table and will fail if it does -not exist yet. +`token_owner_activity.sql`/`transfers.sql`; those views select from the transactions +table and will fail if it does not exist yet. `gsfa_nohot.sql` is an alternative to `gsfa.sql`; do not apply both for the same schema set. `entries.sql` is required for Superbank Fumarole/gRPC source defaults and for PoH entry ingestion from Old Faithful / Jetstreamer. RPC and Bigtable sources do not populate `entries`. diff --git a/ddl/cluster/transfers.sql b/ddl/cluster/transfers.sql new file mode 100644 index 0000000..2637f07 --- /dev/null +++ b/ddl/cluster/transfers.sql @@ -0,0 +1,651 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- +-- Copyright 2025-2026 Triton One Limited. All rights reserved. +-- + +-- Transfer ledger tables used by getTransfersByAddress. +CREATE TABLE IF NOT EXISTS default.transfers_local ON CLUSTER '{cluster}' +( + signature FixedString(64), + slot UInt64, + slot_idx UInt32, + transfer_idx UInt32, + inner_instruction_idx UInt32, + block_time Nullable(Int64), + transfer_type LowCardinality(String), + amount String, + mint Nullable(FixedString(32)), + decimals Nullable(UInt8), + from_user_account Nullable(FixedString(32)), + to_user_account Nullable(FixedString(32)), + from_token_account Nullable(FixedString(32)), + to_token_account Nullable(FixedString(32)), + fee_amount Nullable(String), + INDEX bf_signature signature TYPE bloom_filter(0.01) GRANULARITY 64 +) +ENGINE = ReplacingMergeTree(slot) +PARTITION BY intDiv(slot, 3024000) +PRIMARY KEY (slot, slot_idx, transfer_idx, inner_instruction_idx, transfer_type, signature) +ORDER BY (slot DESC, slot_idx DESC, transfer_idx DESC, inner_instruction_idx, transfer_type, signature) +SETTINGS + allow_experimental_reverse_key = 1, + index_granularity = 512, + index_granularity_bytes = 67108864, + min_bytes_for_wide_part = 10485760, + compress_primary_key = 1, + compress_marks = 1 +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS default.transfers ON CLUSTER '{cluster}' +( + signature FixedString(64), + slot UInt64, + slot_idx UInt32, + transfer_idx UInt32, + inner_instruction_idx UInt32, + block_time Nullable(Int64), + transfer_type LowCardinality(String), + amount String, + mint Nullable(FixedString(32)), + decimals Nullable(UInt8), + from_user_account Nullable(FixedString(32)), + to_user_account Nullable(FixedString(32)), + from_token_account Nullable(FixedString(32)), + to_token_account Nullable(FixedString(32)), + fee_amount Nullable(String), + INDEX bf_signature signature TYPE bloom_filter(0.01) GRANULARITY 64 +) +ENGINE = Distributed('{cluster}', 'default', 'transfers_local', cityHash64(signature)) +AS +SELECT + signature, + slot, + slot_idx, + entry.2 AS transfer_idx, + entry.11 AS inner_instruction_idx, + block_time, + entry.3 AS transfer_type, + entry.8 AS amount, + entry.9 AS mint, + entry.10 AS decimals, + entry.4 AS from_user_account, + entry.5 AS to_user_account, + entry.6 AS from_token_account, + entry.7 AS to_token_account, + entry.12 AS fee_amount +FROM +( + WITH + arrayConcat(tx_account_keys, meta_loaded_addresses_writable, meta_loaded_addresses_readonly) AS account_keys_all, + CAST(base58Decode('11111111111111111111111111111111') AS FixedString(32)) AS system_program, + CAST(base58Decode('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA') AS FixedString(32)) AS token_program, + CAST(base58Decode('TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb') AS FixedString(32)) AS token_2022_program, + CAST(base58Decode('So11111111111111111111111111111111111111111') AS FixedString(32)) AS native_sol_mint, + CAST(base58Decode('So11111111111111111111111111111111111111112') AS FixedString(32)) AS wsol_mint, + arrayMap( + (ix_idx, prog_idx, accts, data) -> + ( + toUInt32(ix_idx - 1), + toUInt32(0), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(tx_instructions_program_id_index), + tx_instructions_program_id_index, + tx_instructions_accounts, + tx_instructions_data + ) AS outer_ixs, + -- Build a normalized inner-instruction list only when metadata includes it. + if( + meta_inner_instructions_present = 1, + -- Collapse each parent instruction's inner instruction array into one flat list. + arrayFlatten( + arrayMap( + -- Keep each inner instruction tied to its outer parent index. + (parent_ix, prog_ids, accts_arr, data_arr) -> + arrayMap( + -- Shape inner instructions to match outer_ixs: parent index, inner index, program id, accounts, data. + (inner_idx, prog_idx, accts, data) -> + ( + toUInt32(parent_ix), + toUInt32(inner_idx), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(prog_ids), + prog_ids, + accts_arr, + data_arr + ), + meta_inner_instructions_index, + meta_inner_instructions_program_id_index, + meta_inner_instructions_accounts, + meta_inner_instructions_data + ) + ), + -- Preserve the tuple type when no inner instructions are present. + CAST([] AS Array(Tuple( + UInt32, + UInt32, + FixedString(32), + Array(UInt8), + String + ))) + ) AS inner_ixs, + -- Treat top-level and inner instructions uniformly for transfer extraction. + arrayConcat(outer_ixs, inner_ixs) AS all_ixs, + -- Convert each normalized instruction into the transfer row fields used below, then keep only + -- successfully parsed transfer-like entries. + arrayFilter( + entry -> entry.1 = 1, + arrayMap( + ix -> + ( + -- filter for valid instruction type + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 5: WithdrawNonceAccount + -- 3: CreateAccountWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 3, 5, 13), + toUInt8(1), + -- 2: Transfer + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (2, 11) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 5, 8)) > 0, + toUInt8(1), + toUInt8(0) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 2, 8)) > 0 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 3, 8)) > 0 + ), + toUInt8(1), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + toUInt8(1), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + toUInt8(1), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toUInt8(1), + toUInt8(0) + ), + toUInt8(0) + ), + ix.1, + -- transfer type + multiIf( + ix.3 = system_program, + 'transfer', + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 12) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + 'transfer', + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + 'mint', + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + 'burn', + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + 'closeAccount', + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + 'changeOwner', + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + 'withdrawWithheldFee', + -- Unreachable + '' + ), + -- Unreachable + '' + ), + -- fromUserAccount + multiIf( + ix.3 = system_program, + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + multiIf( + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + if( + length(ix.4) >= 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14) + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2 + ), + CAST(NULL AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 3, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 3: Transfer + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_pre_token_owner, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toUserAccount + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 3, 5), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 11, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 3)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + if( + reinterpretAsUInt8(substring(ix.5, 3, 1)) = 1, + CAST(substring(ix.5, 4, 32) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- fromTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 8: Burn + -- 9: CloseAccount + -- 12: TransferChecked + -- 15: BurnChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 12: TransferChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- raw transfer amount + multiIf( + ix.3 = system_program, + multiIf( + -- 3: CreateAccountWithSeed + -- [0..4] discriminator, [4..36] base, [36..44] u64 seed length, + -- [44..] seed, then u64 amount / u64 space / owner. + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 3, + toString(reinterpretAsUInt64(substring( + ix.5, + 45 + reinterpretAsUInt64(substring(ix.5, 37, 8)), + 8 + ))), + -- 0: CreateAccount + -- 2: Transfer + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 5, 11, 13), + toString(reinterpretAsUInt64(substring(ix.5, 5, 8))), + '0' + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15), + toString(reinterpretAsUInt64(substring(ix.5, 2, 8))), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + toString( + reinterpretAsUInt64(substring(ix.5, 3, 8)) + - reinterpretAsUInt64(substring(ix.5, 12, 8)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + multiIf( + nullIf( + arrayElement( + meta_pre_token_mint, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) = wsol_mint, + toString( + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ), + toString( + toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 2) + 1)) + - toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 2) + 1)) + ) + ), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toString( + abs( + toUInt64OrZero( + arrayElement( + meta_post_token_amount, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ) + ) + - toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) + ) + ) + ), + '0' + ), + '0' + ), + -- mint address + multiIf( + ix.3 = system_program, + CAST(native_sol_mint AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 3, + CAST( + arrayElement( + meta_post_token_mint, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 1)) + ) AS Nullable(FixedString(32)) + ), + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(native_sol_mint AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- mint decimals + multiIf( + ix.3 = system_program, + CAST(9 AS Nullable(UInt8)), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7), + CAST( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) AS Nullable(UInt8) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(9 AS Nullable(UInt8)), + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (12, 14, 15), + CAST(reinterpretAsUInt8(substring(ix.5, 10, 1)) AS Nullable(UInt8)), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(reinterpretAsUInt8(substring(ix.5, 11, 1)) AS Nullable(UInt8)), + -- 8: Burn + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 8 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + nullIf( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('UInt8') + ) AS Nullable(UInt8) + ), + CAST(NULL AS Nullable(UInt8)) + ), + CAST(NULL AS Nullable(UInt8)) + ), + ix.2, + multiIf( + ix.3 IN (token_program, token_2022_program) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + AND reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(toString(reinterpretAsUInt64(substring(ix.5, 12, 8))) AS Nullable(String)), + CAST(NULL AS Nullable(String)) + ) + ), + all_ixs + ) + ) AS parsed_entries + SELECT + signature, + slot, + slot_idx, + block_time, + parsed_entries AS transfer_entries + FROM default.transactions_local + WHERE meta_status_ok = 1 +) +ARRAY JOIN transfer_entries AS entry; diff --git a/ddl/local/transfers.sql b/ddl/local/transfers.sql new file mode 100644 index 0000000..1102117 --- /dev/null +++ b/ddl/local/transfers.sql @@ -0,0 +1,631 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- +-- Copyright 2025-2026 Triton One Limited. All rights reserved. +-- + +-- Local-only canonical transfer ledger used by getTransfersByAddress (single-node deployments). +-- Parses outer and inner instructions from successful transactions only. +-- Query addresses must be wallet owners (not ATAs); token owners come from meta token balance metadata. +CREATE MATERIALIZED VIEW IF NOT EXISTS default.transfers +( + signature FixedString(64), + slot UInt64, + slot_idx UInt32, + transfer_idx UInt32, + inner_instruction_idx UInt32, + block_time Nullable(Int64), + transfer_type LowCardinality(String), + amount String, + mint Nullable(FixedString(32)), + decimals Nullable(UInt8), + from_user_account Nullable(FixedString(32)), + to_user_account Nullable(FixedString(32)), + from_token_account Nullable(FixedString(32)), + to_token_account Nullable(FixedString(32)), + fee_amount Nullable(String), + INDEX bf_signature signature TYPE bloom_filter(0.01) GRANULARITY 64 +) +ENGINE = ReplacingMergeTree(slot) +PARTITION BY intDiv(slot, 3024000) +PRIMARY KEY (slot, slot_idx, transfer_idx, inner_instruction_idx, transfer_type, signature) +ORDER BY (slot DESC, slot_idx DESC, transfer_idx DESC, inner_instruction_idx, transfer_type, signature) +SETTINGS + allow_experimental_reverse_key = 1, + index_granularity = 512, + index_granularity_bytes = 67108864, + min_bytes_for_wide_part = 10485760, + compress_primary_key = 1, + compress_marks = 1 +AS +SELECT + signature, + slot, + slot_idx, + entry.2 AS transfer_idx, + entry.11 AS inner_instruction_idx, + block_time, + entry.3 AS transfer_type, + entry.8 AS amount, + entry.9 AS mint, + entry.10 AS decimals, + entry.4 AS from_user_account, + entry.5 AS to_user_account, + entry.6 AS from_token_account, + entry.7 AS to_token_account, + entry.12 AS fee_amount +FROM +( + WITH + arrayConcat(tx_account_keys, meta_loaded_addresses_writable, meta_loaded_addresses_readonly) AS account_keys_all, + CAST(base58Decode('11111111111111111111111111111111') AS FixedString(32)) AS system_program, + CAST(base58Decode('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA') AS FixedString(32)) AS token_program, + CAST(base58Decode('TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb') AS FixedString(32)) AS token_2022_program, + CAST(base58Decode('So11111111111111111111111111111111111111111') AS FixedString(32)) AS native_sol_mint, + CAST(base58Decode('So11111111111111111111111111111111111111112') AS FixedString(32)) AS wsol_mint, + arrayMap( + (ix_idx, prog_idx, accts, data) -> + ( + toUInt32(ix_idx - 1), + toUInt32(0), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(tx_instructions_program_id_index), + tx_instructions_program_id_index, + tx_instructions_accounts, + tx_instructions_data + ) AS outer_ixs, + -- Build a normalized inner-instruction list only when metadata includes it. + if( + meta_inner_instructions_present = 1, + -- Collapse each parent instruction's inner instruction array into one flat list. + arrayFlatten( + arrayMap( + -- Keep each inner instruction tied to its outer parent index. + (parent_ix, prog_ids, accts_arr, data_arr) -> + arrayMap( + -- Shape inner instructions to match outer_ixs: parent index, inner index, program id, accounts, data. + (inner_idx, prog_idx, accts, data) -> + ( + toUInt32(parent_ix), + toUInt32(inner_idx), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(prog_ids), + prog_ids, + accts_arr, + data_arr + ), + meta_inner_instructions_index, + meta_inner_instructions_program_id_index, + meta_inner_instructions_accounts, + meta_inner_instructions_data + ) + ), + -- Preserve the tuple type when no inner instructions are present. + CAST([] AS Array(Tuple( + UInt32, + UInt32, + FixedString(32), + Array(UInt8), + String + ))) + ) AS inner_ixs, + -- Treat top-level and inner instructions uniformly for transfer extraction. + arrayConcat(outer_ixs, inner_ixs) AS all_ixs, + -- Convert each normalized instruction into the transfer row fields used below, then keep only + -- successfully parsed transfer-like entries. + arrayFilter( + entry -> entry.1 = 1, + arrayMap( + ix -> + ( + -- filter for valid instruction type + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 5: WithdrawNonceAccount + -- 3: CreateAccountWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 3, 5, 13), + toUInt8(1), + -- 2: Transfer + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (2, 11) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 5, 8)) > 0, + toUInt8(1), + toUInt8(0) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 2, 8)) > 0 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 3, 8)) > 0 + ), + toUInt8(1), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + toUInt8(1), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + toUInt8(1), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toUInt8(1), + toUInt8(0) + ), + toUInt8(0) + ), + ix.1, + -- transfer type + multiIf( + ix.3 = system_program, + 'transfer', + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 12) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + 'transfer', + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + 'mint', + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + 'burn', + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + 'closeAccount', + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + 'changeOwner', + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + 'withdrawWithheldFee', + -- Unreachable + '' + ), + -- Unreachable + '' + ), + -- fromUserAccount + multiIf( + ix.3 = system_program, + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + multiIf( + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + if( + length(ix.4) >= 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14) + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2 + ), + CAST(NULL AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 3, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 3: Transfer + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_pre_token_owner, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toUserAccount + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 3, 5), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 11, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 3)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + if( + reinterpretAsUInt8(substring(ix.5, 3, 1)) = 1, + CAST(substring(ix.5, 4, 32) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- fromTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 8: Burn + -- 9: CloseAccount + -- 12: TransferChecked + -- 15: BurnChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 12: TransferChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- raw transfer amount + multiIf( + ix.3 = system_program, + multiIf( + -- 3: CreateAccountWithSeed + -- [0..4] discriminator, [4..36] base, [36..44] u64 seed length, + -- [44..] seed, then u64 amount / u64 space / owner. + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 3, + toString(reinterpretAsUInt64(substring( + ix.5, + 45 + reinterpretAsUInt64(substring(ix.5, 37, 8)), + 8 + ))), + -- 0: CreateAccount + -- 2: Transfer + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 5, 11, 13), + toString(reinterpretAsUInt64(substring(ix.5, 5, 8))), + '0' + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15), + toString(reinterpretAsUInt64(substring(ix.5, 2, 8))), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + toString( + reinterpretAsUInt64(substring(ix.5, 3, 8)) + - reinterpretAsUInt64(substring(ix.5, 12, 8)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + multiIf( + nullIf( + arrayElement( + meta_pre_token_mint, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) = wsol_mint, + toString( + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ), + toString( + toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 2) + 1)) + - toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 2) + 1)) + ) + ), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toString( + abs( + toUInt64OrZero( + arrayElement( + meta_post_token_amount, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ) + ) + - toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) + ) + ) + ), + '0' + ), + '0' + ), + -- mint address + multiIf( + ix.3 = system_program, + CAST(native_sol_mint AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 3, + CAST( + arrayElement( + meta_post_token_mint, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 1)) + ) AS Nullable(FixedString(32)) + ), + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(native_sol_mint AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- mint decimals + multiIf( + ix.3 = system_program, + CAST(9 AS Nullable(UInt8)), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7), + CAST( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) AS Nullable(UInt8) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(9 AS Nullable(UInt8)), + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (12, 14, 15), + CAST(reinterpretAsUInt8(substring(ix.5, 10, 1)) AS Nullable(UInt8)), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(reinterpretAsUInt8(substring(ix.5, 11, 1)) AS Nullable(UInt8)), + -- 8: Burn + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 8 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + nullIf( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('UInt8') + ) AS Nullable(UInt8) + ), + CAST(NULL AS Nullable(UInt8)) + ), + CAST(NULL AS Nullable(UInt8)) + ), + ix.2, + multiIf( + ix.3 IN (token_program, token_2022_program) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + AND reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(toString(reinterpretAsUInt64(substring(ix.5, 12, 8))) AS Nullable(String)), + CAST(NULL AS Nullable(String)) + ) + ), + all_ixs + ) + ) AS parsed_entries + SELECT + signature, + slot, + slot_idx, + block_time, + parsed_entries AS transfer_entries + FROM default.transactions + WHERE meta_status_ok = 1 +) +ARRAY JOIN transfer_entries AS entry; diff --git a/ddl/replicated/transfers.sql b/ddl/replicated/transfers.sql new file mode 100644 index 0000000..1b24431 --- /dev/null +++ b/ddl/replicated/transfers.sql @@ -0,0 +1,651 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- +-- Copyright 2025-2026 Triton One Limited. All rights reserved. +-- + +-- Transfer ledger tables with replicated shard-local storage used by getTransfersByAddress. +CREATE TABLE IF NOT EXISTS default.transfers_local ON CLUSTER '{cluster}' +( + signature FixedString(64), + slot UInt64, + slot_idx UInt32, + transfer_idx UInt32, + inner_instruction_idx UInt32, + block_time Nullable(Int64), + transfer_type LowCardinality(String), + amount String, + mint Nullable(FixedString(32)), + decimals Nullable(UInt8), + from_user_account Nullable(FixedString(32)), + to_user_account Nullable(FixedString(32)), + from_token_account Nullable(FixedString(32)), + to_token_account Nullable(FixedString(32)), + fee_amount Nullable(String), + INDEX bf_signature signature TYPE bloom_filter(0.01) GRANULARITY 64 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{cluster}/{database}/{table}/{shard}', '{replica}', slot) +PARTITION BY intDiv(slot, 3024000) +PRIMARY KEY (slot, slot_idx, transfer_idx, inner_instruction_idx, transfer_type, signature) +ORDER BY (slot DESC, slot_idx DESC, transfer_idx DESC, inner_instruction_idx, transfer_type, signature) +SETTINGS + allow_experimental_reverse_key = 1, + index_granularity = 512, + index_granularity_bytes = 67108864, + min_bytes_for_wide_part = 10485760, + compress_primary_key = 1, + compress_marks = 1 +; + +CREATE MATERIALIZED VIEW IF NOT EXISTS default.transfers ON CLUSTER '{cluster}' +( + signature FixedString(64), + slot UInt64, + slot_idx UInt32, + transfer_idx UInt32, + inner_instruction_idx UInt32, + block_time Nullable(Int64), + transfer_type LowCardinality(String), + amount String, + mint Nullable(FixedString(32)), + decimals Nullable(UInt8), + from_user_account Nullable(FixedString(32)), + to_user_account Nullable(FixedString(32)), + from_token_account Nullable(FixedString(32)), + to_token_account Nullable(FixedString(32)), + fee_amount Nullable(String), + INDEX bf_signature signature TYPE bloom_filter(0.01) GRANULARITY 64 +) +ENGINE = Distributed('{cluster}', 'default', 'transfers_local', cityHash64(signature)) +AS +SELECT + signature, + slot, + slot_idx, + entry.2 AS transfer_idx, + entry.11 AS inner_instruction_idx, + block_time, + entry.3 AS transfer_type, + entry.8 AS amount, + entry.9 AS mint, + entry.10 AS decimals, + entry.4 AS from_user_account, + entry.5 AS to_user_account, + entry.6 AS from_token_account, + entry.7 AS to_token_account, + entry.12 AS fee_amount +FROM +( + WITH + arrayConcat(tx_account_keys, meta_loaded_addresses_writable, meta_loaded_addresses_readonly) AS account_keys_all, + CAST(base58Decode('11111111111111111111111111111111') AS FixedString(32)) AS system_program, + CAST(base58Decode('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA') AS FixedString(32)) AS token_program, + CAST(base58Decode('TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb') AS FixedString(32)) AS token_2022_program, + CAST(base58Decode('So11111111111111111111111111111111111111111') AS FixedString(32)) AS native_sol_mint, + CAST(base58Decode('So11111111111111111111111111111111111111112') AS FixedString(32)) AS wsol_mint, + arrayMap( + (ix_idx, prog_idx, accts, data) -> + ( + toUInt32(ix_idx - 1), + toUInt32(0), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(tx_instructions_program_id_index), + tx_instructions_program_id_index, + tx_instructions_accounts, + tx_instructions_data + ) AS outer_ixs, + -- Build a normalized inner-instruction list only when metadata includes it. + if( + meta_inner_instructions_present = 1, + -- Collapse each parent instruction's inner instruction array into one flat list. + arrayFlatten( + arrayMap( + -- Keep each inner instruction tied to its outer parent index. + (parent_ix, prog_ids, accts_arr, data_arr) -> + arrayMap( + -- Shape inner instructions to match outer_ixs: parent index, inner index, program id, accounts, data. + (inner_idx, prog_idx, accts, data) -> + ( + toUInt32(parent_ix), + toUInt32(inner_idx), + arrayElement(account_keys_all, prog_idx + 1), + accts, + data + ), + arrayEnumerate(prog_ids), + prog_ids, + accts_arr, + data_arr + ), + meta_inner_instructions_index, + meta_inner_instructions_program_id_index, + meta_inner_instructions_accounts, + meta_inner_instructions_data + ) + ), + -- Preserve the tuple type when no inner instructions are present. + CAST([] AS Array(Tuple( + UInt32, + UInt32, + FixedString(32), + Array(UInt8), + String + ))) + ) AS inner_ixs, + -- Treat top-level and inner instructions uniformly for transfer extraction. + arrayConcat(outer_ixs, inner_ixs) AS all_ixs, + -- Convert each normalized instruction into the transfer row fields used below, then keep only + -- successfully parsed transfer-like entries. + arrayFilter( + entry -> entry.1 = 1, + arrayMap( + ix -> + ( + -- filter for valid instruction type + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 5: WithdrawNonceAccount + -- 3: CreateAccountWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 3, 5, 13), + toUInt8(1), + -- 2: Transfer + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (2, 11) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 5, 8)) > 0, + toUInt8(1), + toUInt8(0) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15) + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 2, 8)) > 0 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + -- Ignore zero-amount transfers. + AND reinterpretAsUInt64(substring(ix.5, 3, 8)) > 0 + ), + toUInt8(1), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + toUInt8(1), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + toUInt8(1), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toUInt8(1), + toUInt8(0) + ), + toUInt8(0) + ), + ix.1, + -- transfer type + multiIf( + ix.3 = system_program, + 'transfer', + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 12) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + 'transfer', + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + 'mint', + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + 'burn', + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + 'closeAccount', + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + 'changeOwner', + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + 'withdrawWithheldFee', + -- Unreachable + '' + ), + -- Unreachable + '' + ), + -- fromUserAccount + multiIf( + ix.3 = system_program, + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + multiIf( + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + if( + length(ix.4) >= 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14) + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2 + ), + CAST(NULL AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 3, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 3: Transfer + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_pre_token_owner, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toUserAccount + multiIf( + ix.3 = system_program, + multiIf( + -- 0: CreateAccount + -- 2: Transfer + -- 3: CreateAccountWithSeed + -- 5: WithdrawNonceAccount + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 3, 5), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 11: TransferWithSeed + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 11, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 13, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 12: TransferChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST( + -- Infer from transaction metadata token balances + nullIf( + arrayElement( + meta_post_token_owner, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 3)) + ), + defaultValueOfTypeName('FixedString(32)') + ) AS Nullable(FixedString(32)) + ), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 6: SetAuthority + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 2, + if( + reinterpretAsUInt8(substring(ix.5, 3, 1)) = 1, + CAST(substring(ix.5, 4, 32) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- fromTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 8: Burn + -- 9: CloseAccount + -- 12: TransferChecked + -- 15: BurnChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- toTokenAccount + multiIf( + ix.3 = system_program, + CAST(NULL AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 14: MintToChecked + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 14) + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 12: TransferChecked + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 12 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), + CAST(NULL AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- raw transfer amount + multiIf( + ix.3 = system_program, + multiIf( + -- 3: CreateAccountWithSeed + -- [0..4] discriminator, [4..36] base, [36..44] u64 seed length, + -- [44..] seed, then u64 amount / u64 space / owner. + reinterpretAsUInt32(substring(ix.5, 1, 4)) = 3, + toString(reinterpretAsUInt64(substring( + ix.5, + 45 + reinterpretAsUInt64(substring(ix.5, 37, 8)), + 8 + ))), + -- 0: CreateAccount + -- 2: Transfer + -- 5: WithdrawNonceAccount + -- 11: TransferWithSeed + -- 13: CreateAccountAllowPrefund + reinterpretAsUInt32(substring(ix.5, 1, 4)) IN (0, 2, 5, 11, 13), + toString(reinterpretAsUInt64(substring(ix.5, 5, 8))), + '0' + ), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + -- 8: Burn + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7, 8, 12, 14, 15), + toString(reinterpretAsUInt64(substring(ix.5, 2, 8))), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + toString( + reinterpretAsUInt64(substring(ix.5, 3, 8)) + - reinterpretAsUInt64(substring(ix.5, 12, 8)) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + multiIf( + nullIf( + arrayElement( + meta_pre_token_mint, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('FixedString(32)') + ) = wsol_mint, + toString( + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ), + toString( + toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 2) + 1)) + - toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 2) + 1)) + ) + ), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + toString( + abs( + toUInt64OrZero( + arrayElement( + meta_post_token_amount, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 2)) + ) + ) + - toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) + ) + ) + ), + '0' + ), + '0' + ), + -- mint address + multiIf( + ix.3 = system_program, + CAST(native_sol_mint AS Nullable(FixedString(32))), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 3, + CAST( + arrayElement( + meta_post_token_mint, + indexOf(meta_post_token_account_index, arrayElement(ix.4, 1)) + ) AS Nullable(FixedString(32)) + ), + -- 7: MintTo + -- 14: MintToChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (7, 14), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 8: Burn + -- 12: TransferChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 12, 15) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 + ), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(native_sol_mint AS Nullable(FixedString(32))), + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), + CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + CAST(NULL AS Nullable(FixedString(32))) + ), + CAST(NULL AS Nullable(FixedString(32))) + ), + -- mint decimals + multiIf( + ix.3 = system_program, + CAST(9 AS Nullable(UInt8)), + ix.3 IN (token_program, token_2022_program), + multiIf( + -- 3: Transfer + -- 7: MintTo + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 7), + CAST( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 2)) + ) AS Nullable(UInt8) + ), + -- 9: CloseAccount + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + CAST(9 AS Nullable(UInt8)), + -- 12: TransferChecked + -- 14: MintToChecked + -- 15: BurnChecked + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (12, 14, 15), + CAST(reinterpretAsUInt8(substring(ix.5, 10, 1)) AS Nullable(UInt8)), + -- 26: TransferFee + -- 1: TransferCheckedWithFee + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(reinterpretAsUInt8(substring(ix.5, 11, 1)) AS Nullable(UInt8)), + -- 8: Burn + -- 26: TransferFee + -- 2: WithdrawWithheldTokensFromMint + -- 3: WithdrawWithheldTokensFromAccounts + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 8 + OR ( + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3) + ), + CAST( + nullIf( + arrayElement( + meta_pre_token_decimals, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ), + defaultValueOfTypeName('UInt8') + ) AS Nullable(UInt8) + ), + CAST(NULL AS Nullable(UInt8)) + ), + CAST(NULL AS Nullable(UInt8)) + ), + ix.2, + multiIf( + ix.3 IN (token_program, token_2022_program) + -- 26: TransferFee + -- 1: TransferCheckedWithFee + AND reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + CAST(toString(reinterpretAsUInt64(substring(ix.5, 12, 8))) AS Nullable(String)), + CAST(NULL AS Nullable(String)) + ) + ), + all_ixs + ) + ) AS parsed_entries + SELECT + signature, + slot, + slot_idx, + block_time, + parsed_entries AS transfer_entries + FROM default.transactions_local + WHERE meta_status_ok = 1 +) +ARRAY JOIN transfer_entries AS entry; diff --git a/deploy/k8s/11-clickhouse-ddl-job.yaml b/deploy/k8s/11-clickhouse-ddl-job.yaml index c4d5b64..151ad2f 100644 --- a/deploy/k8s/11-clickhouse-ddl-job.yaml +++ b/deploy/k8s/11-clickhouse-ddl-job.yaml @@ -50,6 +50,7 @@ spec: clickhouse-client --host clickhouse --user "${CLICKHOUSE_USER}" --password "${CLICKHOUSE_PASSWORD}" --multiquery < /ddl/gsfa_hot.sql clickhouse-client --host clickhouse --user "${CLICKHOUSE_USER}" --password "${CLICKHOUSE_PASSWORD}" --multiquery < /ddl/signatures.sql clickhouse-client --host clickhouse --user "${CLICKHOUSE_USER}" --password "${CLICKHOUSE_PASSWORD}" --multiquery < /ddl/token_owner_activity.sql + clickhouse-client --host clickhouse --user "${CLICKHOUSE_USER}" --password "${CLICKHOUSE_PASSWORD}" --multiquery < /ddl/transfers.sql echo "DDL applied." volumeMounts: - name: ddl diff --git a/scripts/ci/release-e2e.sh b/scripts/ci/release-e2e.sh index 5a456d9..52b36cd 100755 --- a/scripts/ci/release-e2e.sh +++ b/scripts/ci/release-e2e.sh @@ -246,6 +246,7 @@ export SIGNATURE_FILE="${POOL_DIR}/signatures.txt" export SLOT_FILE="${POOL_DIR}/slots.txt" export HOT_ADDRESS="${HOT_ADDRESS:-$(head -n 1 "${ADDRESS_FILE}")}" export TFA_REFERENCE_RPC_URL="${TFA_REFERENCE_RPC_URL:-${RPC_URL}}" +export TBA_REFERENCE_RPC_URL="${TBA_REFERENCE_RPC_URL:-${RPC_URL}}" if [[ "${SUPERBANK_E2E_REFERENCE_RPC_URL}" == "self" ]]; then export REFERENCE_RPC_URL="${RPC_URL}" diff --git a/scripts/dev/run-local-rpc.sh b/scripts/dev/run-local-rpc.sh index 69125a2..f9fd47f 100755 --- a/scripts/dev/run-local-rpc.sh +++ b/scripts/dev/run-local-rpc.sh @@ -82,6 +82,7 @@ fi : "${CLICKHOUSE_GSFA_TABLE:=default.gsfa}" : "${CLICKHOUSE_SIGNATURE_STATUSES_TABLE:=}" : "${CLICKHOUSE_TOKEN_OWNER_ACTIVITY_TABLE:=default.token_owner_activity}" +: "${CLICKHOUSE_TRANSFERS_TABLE:=default.transfers}" : "${CLICKHOUSE_GSFA_HOT_TABLE:=default.gsfa_hot}" : "${CLICKHOUSE_GSFA_HOT_LOCAL_TABLE:=default.gsfa_hot_local}" : "${CLICKHOUSE_GSFA_HOT_ADDRESSES:=}" @@ -95,6 +96,7 @@ fi : "${CLICKHOUSE_GSFA_LOCAL_TABLE:=}" : "${CLICKHOUSE_SIGNATURES_LOCAL_TABLE:=}" : "${CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE:=}" +: "${CLICKHOUSE_TRANSFERS_LOCAL_TABLE:=}" : "${CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE:=}" : "${CLICKHOUSE_BLOCKS_METADATA_LOCAL_TABLE:=}" @@ -132,6 +134,9 @@ fi if [[ -z "${CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE:-}" ]]; then CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE="$(derive_local_table "${CLICKHOUSE_TOKEN_OWNER_ACTIVITY_TABLE}")" fi +if [[ -z "${CLICKHOUSE_TRANSFERS_LOCAL_TABLE:-}" ]]; then + CLICKHOUSE_TRANSFERS_LOCAL_TABLE="$(derive_local_table "${CLICKHOUSE_TRANSFERS_TABLE}")" +fi if [[ -z "${CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE:-}" ]]; then CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE="$(derive_local_table "${CLICKHOUSE_TRANSACTION_TABLE}")" fi @@ -184,6 +189,7 @@ export CLICKHOUSE_GSFA_HOT_LOCAL_TABLE export CLICKHOUSE_GSFA_HOT_ADDRESSES export CLICKHOUSE_SIGNATURE_STATUSES_TABLE export CLICKHOUSE_TOKEN_OWNER_ACTIVITY_TABLE +export CLICKHOUSE_TRANSFERS_TABLE export CLICKHOUSE_TRANSACTION_TABLE export CLICKHOUSE_BLOCKS_METADATA_TABLE export MAX_SIGNATURES_LIMIT @@ -201,6 +207,7 @@ export CLICKHOUSE_STARTUP_TABLE_CHECK export CLICKHOUSE_GSFA_LOCAL_TABLE export CLICKHOUSE_SIGNATURES_LOCAL_TABLE export CLICKHOUSE_TOKEN_OWNER_ACTIVITY_LOCAL_TABLE +export CLICKHOUSE_TRANSFERS_LOCAL_TABLE export CLICKHOUSE_TRANSACTIONS_LOCAL_TABLE export CLICKHOUSE_BLOCKS_METADATA_LOCAL_TABLE if [[ -n "${CLICKHOUSE_SHARD_HTTP_PORT:-}" ]]; then diff --git a/scripts/test/run-k6.sh b/scripts/test/run-k6.sh index 76ba1fe..00adb96 100755 --- a/scripts/test/run-k6.sh +++ b/scripts/test/run-k6.sh @@ -20,6 +20,7 @@ Environment (common): RPC_URL Superbank RPC endpoint (default: http://localhost:8899) REFERENCE_RPC_URL Reference RPC endpoint for validation scenarios (optional) TFA_REFERENCE_RPC_URL Reference RPC endpoint for getTransactionsForAddress validation (optional) + TBA_REFERENCE_RPC_URL Reference RPC endpoint for getTransfersByAddress validation (optional) VALIDATE_LATEST_BLOCKHASH Set to 0 to skip reference isBlockhashValid validation (default: 1) Pools (defaults point at sample data in this repo): @@ -260,6 +261,18 @@ run_validation_suite() { else skip "validate:getTransactionsForAddress" "TFA_REFERENCE_RPC_URL not set" fi + + if [[ -n "${TBA_REFERENCE_RPC_URL:-}" ]]; then + local original_reference_rpc_url="${REFERENCE_RPC_URL}" + REFERENCE_RPC_URL="${TBA_REFERENCE_RPC_URL}" + export REFERENCE_RPC_URL + run_k6 "validate:getTransfersByAddress" \ + "tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js" + REFERENCE_RPC_URL="${original_reference_rpc_url}" + export REFERENCE_RPC_URL + else + skip "validate:getTransfersByAddress" "TBA_REFERENCE_RPC_URL not set" + fi fi run_k6 "validate:json-rpc batch protocol" \ diff --git a/tests/k6/README.md b/tests/k6/README.md index bbee2d4..c97e4e6 100644 --- a/tests/k6/README.md +++ b/tests/k6/README.md @@ -1,7 +1,7 @@ # superbank-rpc k6 Load Tests Comprehensive load testing suite for superbank-rpc methods, including stress tests for many supported RPCs. -Includes basic tests plus validation tests for `getSignaturesForAddress`, `getTransaction`, `getBlock`, `getTransactionCount`, `getInflationReward`, a dedicated endpoint-comparison scenario for `getTransactionsForAddress`, a disk-cache parity scenario (`superbank-rpc-disk-cache-parity.js`) that diffs a disk-cache-enabled target against a reference across every method the disk tier serves, and a disk-cache performance comparison scenario (`superbank-rpc-disk-cache-compare.js`) that reports latency deltas and speedup ratios versus a non-disk reference. +Includes basic tests plus validation tests for `getSignaturesForAddress`, `getTransaction`, `getBlock`, `getTransactionCount`, `getInflationReward`, dedicated endpoint-comparison scenarios for `getTransactionsForAddress` and `getTransfersByAddress`, a disk-cache parity scenario (`superbank-rpc-disk-cache-parity.js`) that diffs a disk-cache-enabled target against a reference across every method the disk tier serves, and a disk-cache performance comparison scenario (`superbank-rpc-disk-cache-compare.js`) that reports latency deltas and speedup ratios versus a non-disk reference. ## Quick Start @@ -72,6 +72,12 @@ k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transactions-for -e REFERENCE_RPC_URL=http://localhost:8898 \ -e ADDRESS_FILE=./tests/k6/data/pools/addresses.txt +# Validation + latency comparison for getTransfersByAddress +k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js \ + -e RPC_URL=http://localhost:8899 \ + -e REFERENCE_RPC_URL=http://localhost:8898 \ + -e ADDRESS_FILE=./tests/k6/data/pools/addresses.txt + # Validation test for JSON-RPC batch semantics (no reference RPC required) k6 run tests/k6/scenarios/validation/superbank-rpc-validate-batch.js -e RPC_URL=http://localhost:8899 @@ -195,6 +201,11 @@ Compare Superbank RPC responses to a reference RPC. Responses must match exactly The `getTransactionsForAddress` comparison scenario is intended for Superbank-to-Superbank comparisons, not official Solana RPC endpoints, because the method is custom. +The `getTransfersByAddress` method was first implemented by Helius and inspired Superbank's +method shape, but Superbank's implementation is not backward-compatible with Helius. The same +request parameters can return different response projections, so the exact-response validation +scenario should use another Superbank-compatible endpoint as its reference. + ```bash # getSignaturesForAddress validation k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-signatures.js \ @@ -229,6 +240,12 @@ k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transactions-for -e ADDRESS_FILE=./tests/k6/data/pools/addresses.txt # This alternates call order per iteration and reports which endpoint is faster by avg latency. +# getTransfersByAddress validation + latency comparison +k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js \ + -e RPC_URL=http://localhost:8899 \ + -e REFERENCE_RPC_URL=http://localhost:8898 \ + -e ADDRESS_FILE=./tests/k6/data/pools/addresses.txt + # Include full transaction payloads and version fields in the comparison k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transactions-for-address.js \ -e RPC_URL=http://localhost:8899 \ @@ -471,6 +488,7 @@ server-side JSON-RPC failures are promoted to HTTP `503`; those responses count | `RPC_URLS` | (none) | Comma/space-separated list of RPC endpoints to rotate per iteration | | `REFERENCE_RPC_URL` | (none) | Reference RPC endpoint URL for validation tests | | `TFA_REFERENCE_RPC_URL` | (none) | Reference Superbank endpoint URL for `getTransactionsForAddress` validation | +| `TBA_REFERENCE_RPC_URL` | (none) | Reference Superbank endpoint URL for `getTransfersByAddress` validation | | `VALIDATE_LATEST_BLOCKHASH` | `1` | Set to `0` to skip `validate:getLatestBlockhash` when no live `isBlockhashValid` reference is available | | `LIMIT` | `25` | `getSignaturesForAddress` limit parameter | | `ADDRESS_FILE` | (none) | Path to file with known Solana addresses | @@ -518,6 +536,17 @@ server-side JSON-RPC failures are promoted to HTTP `503`; those responses count | `TFA_PAGINATION_TOKEN` | (none) | getTransactionsForAddress `paginationToken` | | `TFA_STATUS` | (none) | getTransactionsForAddress `filters.status` | | `TFA_TOKEN_ACCOUNTS` | (none) | getTransactionsForAddress `filters.tokenAccounts` | +| `TRANSFERS_SORT_ORDER` | `desc` | getTransfersByAddress `sortOrder` (`asc`, `desc`) | +| `TRANSFERS_LIMIT` | `100` | getTransfersByAddress `limit` | +| `TRANSFERS_COMMITMENT` | `finalized` | getTransfersByAddress commitment (`confirmed`, `finalized`) | +| `TRANSFERS_MIN_CONTEXT_SLOT` | (none) | getTransfersByAddress `minContextSlot` | +| `TRANSFERS_SOL_MODE` | `merged` | getTransfersByAddress `solMode` (`merged`, `separate`) | +| `TRANSFERS_PAGINATION_TOKEN` | (none) | getTransfersByAddress `paginationToken` (`slot:transactionIdx:instructionIdx:innerInstructionIdx:type`) | +| `TRANSFERS_DIRECTION` | `any` | getTransfersByAddress top-level `direction` (`in`, `out`, `any`) | +| `TRANSFERS_MINT` | (none) | getTransfersByAddress top-level `mint` | +| `TRANSFERS_WITH` | (none) | getTransfersByAddress top-level `with` account | +| `TRANSFERS_AMOUNT_GTE` | (none) | getTransfersByAddress `filters.amount.gte` | +| `TRANSFERS_AMOUNT_LTE` | (none) | getTransfersByAddress `filters.amount.lte` | | `RPC_TIMEOUT` | (none) | Fallback per-request timeout for scenarios that support it | | `HOT_ADDRESS` | (USDC mint) | Hot address for the hot pagination scenarios | | `HOT_LIMIT` | `1000` | Page size for hot pagination | @@ -791,7 +820,8 @@ tests/k6/ │ │ ├── superbank-rpc-validate-get-inflation-reward.js │ │ ├── superbank-rpc-validate-get-latest-blockhash.js │ │ ├── superbank-rpc-validate-get-transaction-count.js -│ │ └── superbank-rpc-validate-get-transactions-for-address.js # TFA endpoint comparison +│ │ ├── superbank-rpc-validate-get-transactions-for-address.js # TFA endpoint comparison +│ │ └── superbank-rpc-validate-get-transfers-by-address.js # TBA endpoint comparison │ ├── stress/ │ │ ├── stress-test.js # Stress test (find breaking point) │ │ ├── stress-test-get-block.js diff --git a/tests/k6/lib/config.js b/tests/k6/lib/config.js index 4c5cd61..6c75834 100644 --- a/tests/k6/lib/config.js +++ b/tests/k6/lib/config.js @@ -132,6 +132,28 @@ export const config = { transactionsForAddressStatus: __ENV.TFA_STATUS || null, transactionsForAddressTokenAccounts: __ENV.TFA_TOKEN_ACCOUNTS || null, + // getTransfersByAddress options + transfersByAddressSortOrder: __ENV.TRANSFERS_SORT_ORDER || 'desc', + transfersByAddressLimit: Number(__ENV.TRANSFERS_LIMIT || 100), + transfersByAddressCommitment: __ENV.TRANSFERS_COMMITMENT || 'finalized', + transfersByAddressSolMode: __ENV.TRANSFERS_SOL_MODE || 'merged', + transfersByAddressMinContextSlot: + __ENV.TRANSFERS_MIN_CONTEXT_SLOT !== undefined + ? Number(__ENV.TRANSFERS_MIN_CONTEXT_SLOT) + : null, + transfersByAddressPaginationToken: __ENV.TRANSFERS_PAGINATION_TOKEN || null, + transfersByAddressDirection: __ENV.TRANSFERS_DIRECTION || 'any', + transfersByAddressMint: __ENV.TRANSFERS_MINT || null, + transfersByAddressWith: __ENV.TRANSFERS_WITH || null, + transfersByAddressAmountGte: + __ENV.TRANSFERS_AMOUNT_GTE !== undefined + ? Number(__ENV.TRANSFERS_AMOUNT_GTE) + : null, + transfersByAddressAmountLte: + __ENV.TRANSFERS_AMOUNT_LTE !== undefined + ? Number(__ENV.TRANSFERS_AMOUNT_LTE) + : null, + // WebSocket signature stream (used by head-cache tests) solanaWsUrl: __ENV.SOLANA_WS_URL || null, wsMention: @@ -239,6 +261,52 @@ export function transactionsForAddressOptions() { return options; } +export function transfersByAddressOptions() { + const options = { + sortOrder: config.transfersByAddressSortOrder, + limit: config.transfersByAddressLimit, + }; + + if (config.transfersByAddressCommitment) { + options.commitment = config.transfersByAddressCommitment; + } + if (config.transfersByAddressSolMode) { + options.solMode = config.transfersByAddressSolMode; + } + if (config.transfersByAddressMinContextSlot !== null) { + options.minContextSlot = config.transfersByAddressMinContextSlot; + } + if (config.transfersByAddressPaginationToken) { + options.paginationToken = config.transfersByAddressPaginationToken; + } + if (config.transfersByAddressDirection) { + options.direction = config.transfersByAddressDirection; + } + if (config.transfersByAddressMint) { + options.mint = config.transfersByAddressMint; + } + if (config.transfersByAddressWith) { + options.with = config.transfersByAddressWith; + } + + const filters = {}; + const amount = {}; + if (config.transfersByAddressAmountGte !== null) { + amount.gte = config.transfersByAddressAmountGte; + } + if (config.transfersByAddressAmountLte !== null) { + amount.lte = config.transfersByAddressAmountLte; + } + if (Object.keys(amount).length > 0) { + filters.amount = amount; + } + if (Object.keys(filters).length > 0) { + options.filters = filters; + } + + return options; +} + export function parseNonNegativeIntEnv(name, defaultValue) { const raw = __ENV[name]; if (raw === undefined || raw === null || raw === '') { diff --git a/tests/k6/lib/rpc.js b/tests/k6/lib/rpc.js index 81a42fe..68a8e14 100644 --- a/tests/k6/lib/rpc.js +++ b/tests/k6/lib/rpc.js @@ -126,6 +126,26 @@ export function makeGetTransactionsForAddressRequest(address, options = {}, requ }); } +/** + * Build a JSON-RPC request payload for getTransfersByAddress + * @param {string} address - Solana address + * @param {object} options - Request options + * @param {number} [requestId] - JSON-RPC request id override + * @returns {string} JSON stringified payload + */ +export function makeGetTransfersByAddressRequest(address, options = {}, requestId = null) { + const params = [address]; + if (options && Object.keys(options).length > 0) { + params.push(options); + } + return JSON.stringify({ + jsonrpc: '2.0', + id: requestId === null ? Math.floor(Math.random() * 1_000_000_000) : requestId, + method: 'getTransfersByAddress', + params, + }); +} + /** * Build a JSON-RPC request payload for getBlockTime * @param {number} slot - Solana slot diff --git a/tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js b/tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js new file mode 100644 index 0000000..4dc3329 --- /dev/null +++ b/tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +// Validation + latency comparison test for superbank-rpc getTransfersByAddress. +// +// Purpose: Compare two endpoints that both implement getTransfersByAddress, +// validate response parity, and report which one is faster. +// +// Usage: +// k6 run tests/k6/scenarios/validation/superbank-rpc-validate-get-transfers-by-address.js \ +// -e RPC_URL=http://localhost:8899 \ +// -e REFERENCE_RPC_URL=http://localhost:8898 \ +// -e ADDRESS_FILE=./tests/k6/data/pools/addresses.txt + +import { check } from 'k6'; +import { Counter, Rate, Trend } from 'k6/metrics'; +import { config, scenarios, transfersByAddressOptions } from '../../lib/config.js'; +import { initAddressPool, randomAddress } from '../../lib/addresses.js'; +import { deepEqualWithLogSuperset, summarizeJson } from '../../lib/compare.js'; +import { executeRequest, makeGetTransfersByAddressRequest } from '../../lib/rpc.js'; + +if (!config.referenceRpcUrl) { + throw new Error('REFERENCE_RPC_URL is required for validation tests.'); +} + +const addressPool = initAddressPool(); +const logMismatches = __ENV.VALIDATION_LOG_MISMATCHES !== '0'; + +const primaryLatency = new Trend('transfers_compare_primary_latency_ms', true); +const referenceLatency = new Trend('transfers_compare_reference_latency_ms', true); +const latencyDelta = new Trend('transfers_compare_latency_delta_ms', true); + +const primaryRequests = new Counter('transfers_compare_primary_requests_total'); +const primarySuccessful = new Counter('transfers_compare_primary_success_total'); +const primaryErrors = new Counter('transfers_compare_primary_errors_total'); +const referenceRequests = new Counter('transfers_compare_reference_requests_total'); +const referenceSuccessful = new Counter('transfers_compare_reference_success_total'); +const referenceErrors = new Counter('transfers_compare_reference_errors_total'); + +const comparisonsCompared = new Counter('transfers_compare_compared_total'); +const comparisonsSkipped = new Counter('transfers_compare_skipped_total'); +const comparisonsMatched = new Counter('transfers_compare_matches_total'); +const comparisonsMismatched = new Counter('transfers_compare_mismatches_total'); +const comparisonMatchRate = new Rate('transfers_compare_match_rate'); +const primaryFaster = new Counter('transfers_compare_primary_faster_total'); +const referenceFaster = new Counter('transfers_compare_reference_faster_total'); +const comparisonTies = new Counter('transfers_compare_ties_total'); + +export const options = { + vus: scenarios.basic.vus, + duration: scenarios.basic.duration, + thresholds: { + http_req_failed: ['rate==0.0'], + checks: ['rate==1.0'], + }, +}; + +function recordEndpoint(kind, result) { + const latency = kind === 'primary' ? primaryLatency : referenceLatency; + const requests = kind === 'primary' ? primaryRequests : referenceRequests; + const successful = kind === 'primary' ? primarySuccessful : referenceSuccessful; + const errors = kind === 'primary' ? primaryErrors : referenceErrors; + + requests.add(1); + const duration = result?.response?.timings?.duration; + if (typeof duration === 'number' && Number.isFinite(duration)) { + latency.add(duration); + } + + if (result?.response?.status === 200 && result.body && !result.body.error) { + successful.add(1); + } else { + errors.add(1); + } +} + +function recordLatencyComparison(primary, reference) { + const primaryDuration = primary?.response?.timings?.duration; + const referenceDuration = reference?.response?.timings?.duration; + if ( + typeof primaryDuration !== 'number' || + typeof referenceDuration !== 'number' || + !Number.isFinite(primaryDuration) || + !Number.isFinite(referenceDuration) + ) { + return; + } + + const delta = primaryDuration - referenceDuration; + latencyDelta.add(delta); + if (delta < 0) { + primaryFaster.add(1); + } else if (delta > 0) { + referenceFaster.add(1); + } else { + comparisonTies.add(1); + } +} + +function maybeLogMismatch(address, primaryBody, referenceBody) { + if (!logMismatches) { + return; + } + console.error(`getTransfersByAddress mismatch for ${address} (vu ${__VU}, iter ${__ITER})`); + console.error(`Primary: ${summarizeJson(primaryBody)}`); + console.error(`Reference: ${summarizeJson(referenceBody)}`); +} + +function summarizeTrend(data, metricName) { + const values = data.metrics[metricName]?.values; + return { + avg: values?.avg || 0, + p95: values?.['p(95)'] || 0, + min: values?.min || 0, + max: values?.max || 0, + }; +} + +function summarizeCount(data, metricName) { + return data.metrics[metricName]?.values?.count || 0; +} + +export default function () { + const address = randomAddress(); + const requestId = Math.floor(Math.random() * 1_000_000_000); + const payload = makeGetTransfersByAddressRequest( + address, + transfersByAddressOptions(), + requestId + ); + + const primaryFirst = (__ITER + __VU) % 2 === 0; + const first = primaryFirst + ? { kind: 'primary', rpcUrl: config.rpcUrl } + : { kind: 'reference', rpcUrl: config.referenceRpcUrl }; + const second = primaryFirst + ? { kind: 'reference', rpcUrl: config.referenceRpcUrl } + : { kind: 'primary', rpcUrl: config.rpcUrl }; + + const firstResult = executeRequest(payload, { + rpcUrl: first.rpcUrl, + recordMetrics: false, + }); + recordEndpoint(first.kind, firstResult); + + const secondResult = executeRequest(payload, { + rpcUrl: second.rpcUrl, + recordMetrics: false, + }); + recordEndpoint(second.kind, secondResult); + + const primary = first.kind === 'primary' ? firstResult : secondResult; + const reference = first.kind === 'reference' ? firstResult : secondResult; + const basicChecks = check(null, { + 'primary status is 200': () => primary.response.status === 200, + 'reference status is 200': () => reference.response.status === 200, + 'primary response is json': () => primary.body !== null, + 'reference response is json': () => reference.body !== null, + 'primary has no rpc error': () => primary.body && !primary.body.error, + 'reference has no rpc error': () => reference.body && !reference.body.error, + }); + + if (!basicChecks) { + comparisonsSkipped.add(1); + maybeLogMismatch(address, primary.body, reference.body); + return; + } + + recordLatencyComparison(primary, reference); + comparisonsCompared.add(1); + const match = deepEqualWithLogSuperset(primary.body, reference.body); + comparisonMatchRate.add(match); + if (match) { + comparisonsMatched.add(1); + } else { + comparisonsMismatched.add(1); + maybeLogMismatch(address, primary.body, reference.body); + } + + check(null, { + 'responses match': () => match, + }); +} + +export function handleSummary(data) { + return { + stdout: + JSON.stringify( + { + testType: 'validate-get-transfers-by-address', + timestamp: new Date().toISOString(), + config: { + rpcUrl: config.rpcUrl, + referenceRpcUrl: config.referenceRpcUrl, + vus: scenarios.basic.vus, + duration: scenarios.basic.duration, + addressPoolSize: addressPool.length, + sortOrder: config.transfersByAddressSortOrder, + limit: config.transfersByAddressLimit, + commitment: config.transfersByAddressCommitment, + solMode: config.transfersByAddressSolMode, + minContextSlot: config.transfersByAddressMinContextSlot, + paginationToken: config.transfersByAddressPaginationToken, + direction: config.transfersByAddressDirection, + mint: config.transfersByAddressMint, + with: config.transfersByAddressWith, + amountGte: config.transfersByAddressAmountGte, + amountLte: config.transfersByAddressAmountLte, + }, + metrics: { + primary: { + requests: summarizeCount(data, 'transfers_compare_primary_requests_total'), + successful: summarizeCount(data, 'transfers_compare_primary_success_total'), + errors: summarizeCount(data, 'transfers_compare_primary_errors_total'), + latencyMs: summarizeTrend(data, 'transfers_compare_primary_latency_ms'), + }, + reference: { + requests: summarizeCount(data, 'transfers_compare_reference_requests_total'), + successful: summarizeCount(data, 'transfers_compare_reference_success_total'), + errors: summarizeCount(data, 'transfers_compare_reference_errors_total'), + latencyMs: summarizeTrend(data, 'transfers_compare_reference_latency_ms'), + }, + comparison: { + compared: summarizeCount(data, 'transfers_compare_compared_total'), + skipped: summarizeCount(data, 'transfers_compare_skipped_total'), + matches: summarizeCount(data, 'transfers_compare_matches_total'), + mismatches: summarizeCount(data, 'transfers_compare_mismatches_total'), + matchRate: data.metrics.transfers_compare_match_rate?.values?.rate || 0, + primaryFaster: summarizeCount(data, 'transfers_compare_primary_faster_total'), + referenceFaster: summarizeCount(data, 'transfers_compare_reference_faster_total'), + ties: summarizeCount(data, 'transfers_compare_ties_total'), + latencyDeltaMs: summarizeTrend(data, 'transfers_compare_latency_delta_ms'), + }, + }, + }, + null, + 2 + ) + '\n', + }; +} From e83be845060cc6aa639c4bc58a8a5825cb27be7c Mon Sep 17 00:00:00 2001 From: Chii Yuen Date: Tue, 14 Jul 2026 11:01:33 +0800 Subject: [PATCH 2/3] fix: return paginationToken when there is more than limit + 1 results found --- crates/superbank-rpc/README.md | 2 + .../superbank-rpc/src/clickhouse/transfers.rs | 13 +++- .../src/handlers/transactions.rs | 74 ++++++++++++++++--- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/crates/superbank-rpc/README.md b/crates/superbank-rpc/README.md index 943552f..bb01feb 100644 --- a/crates/superbank-rpc/README.md +++ b/crates/superbank-rpc/README.md @@ -72,6 +72,8 @@ Notes: strings (lamports for SOL, token base units for SPL), with `uiAmount` derived from decimals. The index is derived from pre/post balance deltas, so the opposite user account is nullable when no counterparty can be inferred. This method requires the transfers table (see below). + Response `paginationToken` is non-null only when another matching page exists after all filters + are applied; the token points to the last row returned by the current page. ## ClickHouse schemas diff --git a/crates/superbank-rpc/src/clickhouse/transfers.rs b/crates/superbank-rpc/src/clickhouse/transfers.rs index 8d63575..ec77dce 100644 --- a/crates/superbank-rpc/src/clickhouse/transfers.rs +++ b/crates/superbank-rpc/src/clickhouse/transfers.rs @@ -254,7 +254,7 @@ fn build_transfers_by_address_query( {settings_clause}", body = body, order = order, - limit = query.limit, + limit = query.limit.saturating_add(1), settings_clause = settings_clause, )) } @@ -430,4 +430,15 @@ mod tests { "ORDER BY slot DESC, slot_idx DESC, transfer_idx DESC, inner_instruction_idx DESC, transfer_type DESC" )); } + + #[test] + fn query_fetches_one_extra_row_to_detect_next_page() { + let mut query = base_query(SolMode::Separate); + query.limit = 7; + + let sql = build_transfers_by_address_query("default.transfers", &query, "") + .expect("query should build"); + + assert!(sql.contains("LIMIT 8")); + } } diff --git a/crates/superbank-rpc/src/handlers/transactions.rs b/crates/superbank-rpc/src/handlers/transactions.rs index 64b8f89..4d12fe5 100644 --- a/crates/superbank-rpc/src/handlers/transactions.rs +++ b/crates/superbank-rpc/src/handlers/transactions.rs @@ -374,6 +374,33 @@ fn map_transfer_record_to_response( } } +fn transfer_pagination_token(record: &TransferRecord) -> String { + let transfer_type = + TokenTransferTypes::try_from(record.transfer_type.as_str()).unwrap_or_default(); + format!( + "{}:{}:{}:{}:{}", + record.slot, + record.slot_idx, + record.transfer_idx, + record.inner_instruction_idx, + <&'static str>::from(transfer_type) + ) +} + +fn paginate_transfer_records( + mut records: Vec, + limit: u64, +) -> (Vec, Option) { + let limit = limit as usize; + if records.len() <= limit { + return (records, None); + } + + records.truncate(limit); + let pagination_token = records.last().map(transfer_pagination_token); + (records, pagination_token) +} + fn format_ui_amount(amount: &str, decimals: Option) -> String { let decimals = decimals.unwrap_or(0) as usize; if decimals == 0 { @@ -2258,18 +2285,7 @@ pub(crate) async fn handle_get_transfers_by_address( } }; - let pagination_token = records.last().map(|record| { - let transfer_type = - TokenTransferTypes::try_from(record.transfer_type.as_str()).unwrap_or_default(); - format!( - "{}:{}:{}:{}:{}", - record.slot, - record.slot_idx, - record.transfer_idx, - record.inner_instruction_idx, - <&'static str>::from(transfer_type) - ) - }); + let (records, pagination_token) = paginate_transfer_records(records, limit); let confirmation_status = options.commitment.as_deref().unwrap_or("finalized"); let data = records .into_iter() @@ -2606,6 +2622,40 @@ mod tests { assert_eq!(mapped.to_token_account, None); } + #[test] + fn paginate_transfer_records_only_returns_token_when_extra_row_exists() { + fn record(slot_idx: u32) -> TransferRecord { + TransferRecord { + signature: format!("sig-{slot_idx}"), + slot: 10, + slot_idx, + transfer_idx: 2, + inner_instruction_idx: 0, + block_time: None, + transfer_type: "transfer".to_string(), + amount: "1000".to_string(), + mint: Some(NATIVE_SOL_MINT.to_string()), + decimals: Some(9), + from_user_account: None, + to_user_account: None, + from_token_account: None, + to_token_account: None, + } + } + + let (page, token) = paginate_transfer_records(vec![record(3), record(2)], 2); + assert_eq!(page.len(), 2); + assert_eq!(token, None); + + let (page, token) = paginate_transfer_records(vec![record(3), record(2), record(1)], 2); + assert_eq!(page.len(), 2); + assert_eq!( + token.as_deref(), + Some("10:2:2:0:transfer"), + "cursor should point at the last returned row, not the extra lookahead row" + ); + } + #[test] fn format_ui_amount_returns_raw_amount_without_decimals() { assert_eq!(format_ui_amount("1234500", None), "1234500"); From 258a9ae81a7b9ae48bdae6afdc55c0cb804cfbe0 Mon Sep 17 00:00:00 2001 From: Chii Yuen Date: Tue, 14 Jul 2026 13:15:48 +0800 Subject: [PATCH 3/3] feat: add support for WithdrawExcessLamports and UnwrapLamports --- ddl/cluster/transfers.sql | 70 ++++++++++++++++++++++++++++++++++-- ddl/local/transfers.sql | 70 ++++++++++++++++++++++++++++++++++-- ddl/replicated/transfers.sql | 70 ++++++++++++++++++++++++++++++++++-- 3 files changed, 201 insertions(+), 9 deletions(-) diff --git a/ddl/cluster/transfers.sql b/ddl/cluster/transfers.sql index 2637f07..3e07b42 100644 --- a/ddl/cluster/transfers.sql +++ b/ddl/cluster/transfers.sql @@ -194,6 +194,25 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), toUInt8(1), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38 + -- Ignore zero-amount transfers. + AND toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + > toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)), + toUInt8(1), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) > 0, + toUInt8(1), toUInt8(0) ), toUInt8(0) @@ -236,6 +255,17 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), 'withdrawWithheldFee', + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + 'transfer', + -- 45: UnwrapLamports with explicit amount. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + 'transfer', + -- 45: UnwrapLamports without explicit amount closes the token account. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 0, + 'closeAccount', -- Unreachable '' ), @@ -299,7 +329,9 @@ FROM ) AS Nullable(FixedString(32)) ), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -373,7 +405,9 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), CAST(NULL AS Nullable(FixedString(32))), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -398,9 +432,11 @@ FROM -- 9: CloseAccount -- 12: TransferChecked -- 15: BurnChecked + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports -- 26: TransferFee -- 1: TransferCheckedWithFee - reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15, 38, 45) OR ( reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 @@ -529,6 +565,26 @@ FROM ) ) ), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + toString( + toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + - toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)) + ), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45, + toString( + multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) + ), '0' ), '0' @@ -571,6 +627,10 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(native_sol_mint AS Nullable(FixedString(32))), CAST(NULL AS Nullable(FixedString(32))) ), CAST(NULL AS Nullable(FixedString(32))) @@ -621,6 +681,10 @@ FROM defaultValueOfTypeName('UInt8') ) AS Nullable(UInt8) ), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(9 AS Nullable(UInt8)), CAST(NULL AS Nullable(UInt8)) ), CAST(NULL AS Nullable(UInt8)) diff --git a/ddl/local/transfers.sql b/ddl/local/transfers.sql index 1102117..959607c 100644 --- a/ddl/local/transfers.sql +++ b/ddl/local/transfers.sql @@ -174,6 +174,25 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), toUInt8(1), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38 + -- Ignore zero-amount transfers. + AND toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + > toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)), + toUInt8(1), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) > 0, + toUInt8(1), toUInt8(0) ), toUInt8(0) @@ -216,6 +235,17 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), 'withdrawWithheldFee', + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + 'transfer', + -- 45: UnwrapLamports with explicit amount. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + 'transfer', + -- 45: UnwrapLamports without explicit amount closes the token account. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 0, + 'closeAccount', -- Unreachable '' ), @@ -279,7 +309,9 @@ FROM ) AS Nullable(FixedString(32)) ), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -353,7 +385,9 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), CAST(NULL AS Nullable(FixedString(32))), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -378,9 +412,11 @@ FROM -- 9: CloseAccount -- 12: TransferChecked -- 15: BurnChecked + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports -- 26: TransferFee -- 1: TransferCheckedWithFee - reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15, 38, 45) OR ( reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 @@ -509,6 +545,26 @@ FROM ) ) ), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + toString( + toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + - toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)) + ), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45, + toString( + multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) + ), '0' ), '0' @@ -551,6 +607,10 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(native_sol_mint AS Nullable(FixedString(32))), CAST(NULL AS Nullable(FixedString(32))) ), CAST(NULL AS Nullable(FixedString(32))) @@ -601,6 +661,10 @@ FROM defaultValueOfTypeName('UInt8') ) AS Nullable(UInt8) ), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(9 AS Nullable(UInt8)), CAST(NULL AS Nullable(UInt8)) ), CAST(NULL AS Nullable(UInt8)) diff --git a/ddl/replicated/transfers.sql b/ddl/replicated/transfers.sql index 1b24431..de9b6ac 100644 --- a/ddl/replicated/transfers.sql +++ b/ddl/replicated/transfers.sql @@ -194,6 +194,25 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), toUInt8(1), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38 + -- Ignore zero-amount transfers. + AND toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + > toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)), + toUInt8(1), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) > 0, + toUInt8(1), toUInt8(0) ), toUInt8(0) @@ -236,6 +255,17 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), 'withdrawWithheldFee', + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + 'transfer', + -- 45: UnwrapLamports with explicit amount. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + 'transfer', + -- 45: UnwrapLamports without explicit amount closes the token account. + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45 + AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 0, + 'closeAccount', -- Unreachable '' ), @@ -299,7 +329,9 @@ FROM ) AS Nullable(FixedString(32)) ), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 3) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -373,7 +405,9 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (8, 15), CAST(NULL AS Nullable(FixedString(32))), -- 9: CloseAccount - reinterpretAsUInt8(substring(ix.5, 1, 1)) = 9, + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (9, 38, 45), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 2) + 1) AS Nullable(FixedString(32))), -- 6: SetAuthority reinterpretAsUInt8(substring(ix.5, 1, 1)) = 6 @@ -398,9 +432,11 @@ FROM -- 9: CloseAccount -- 12: TransferChecked -- 15: BurnChecked + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports -- 26: TransferFee -- 1: TransferCheckedWithFee - reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15) + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (3, 8, 9, 12, 15, 38, 45) OR ( reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1 @@ -529,6 +565,26 @@ FROM ) ) ), + -- 38: WithdrawExcessLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 38, + toString( + toUInt64(arrayElement(meta_pre_balances, arrayElement(ix.4, 1) + 1)) + - toUInt64(arrayElement(meta_post_balances, arrayElement(ix.4, 1) + 1)) + ), + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) = 45, + toString( + multiIf( + reinterpretAsUInt8(substring(ix.5, 2, 1)) = 1, + reinterpretAsUInt64(substring(ix.5, 3, 8)), + toUInt64OrZero( + arrayElement( + meta_pre_token_amount, + indexOf(meta_pre_token_account_index, arrayElement(ix.4, 1)) + ) + ) + ) + ), '0' ), '0' @@ -571,6 +627,10 @@ FROM reinterpretAsUInt8(substring(ix.5, 1, 1)) = 26 AND reinterpretAsUInt8(substring(ix.5, 2, 1)) IN (2, 3), CAST(arrayElement(account_keys_all, arrayElement(ix.4, 1) + 1) AS Nullable(FixedString(32))), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(native_sol_mint AS Nullable(FixedString(32))), CAST(NULL AS Nullable(FixedString(32))) ), CAST(NULL AS Nullable(FixedString(32))) @@ -621,6 +681,10 @@ FROM defaultValueOfTypeName('UInt8') ) AS Nullable(UInt8) ), + -- 38: WithdrawExcessLamports + -- 45: UnwrapLamports + reinterpretAsUInt8(substring(ix.5, 1, 1)) IN (38, 45), + CAST(9 AS Nullable(UInt8)), CAST(NULL AS Nullable(UInt8)) ), CAST(NULL AS Nullable(UInt8))