diff --git a/.devcontainer/.blockscout_config.example b/.devcontainer/.blockscout_config.example deleted file mode 100644 index 737933b067b6..000000000000 --- a/.devcontainer/.blockscout_config.example +++ /dev/null @@ -1,61 +0,0 @@ -CHAIN_TYPE=ethereum - -ETHEREUM_JSONRPC_VARIANT=geth -ETHEREUM_JSONRPC_TRACE_URL="" - -API_RATE_LIMIT=100 -HEART_BEAT_TIMEOUT=30 -TXS_STATS_DAYS_TO_COMPILE_AT_INIT=2 -INDEXER_MEMORY_LIMIT=6 - -POOL_SIZE=50 -POOL_SIZE_API=50 -ACCOUNT_POOL_SIZE=10 - -INDEXER_DISABLE_EMPTY_BLOCKS_SANITIZER='true' -INDEXER_DISABLE_PENDING_TRANSACTIONS_FETCHER='true' -INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER='true' -INDEXER_DISABLE_BLOCK_REWARD_FETCHER='true' -INDEXER_DISABLE_ADDRESS_COIN_BALANCE_FETCHER='true' -INDEXER_DISABLE_CATALOGED_TOKEN_UPDATER_FETCHER='true' -ETHEREUM_JSONRPC_DISABLE_ARCHIVE_BALANCES='true' -INDEXER_DISABLE_TOKEN_INSTANCE_RETRY_FETCHER='true' -INDEXER_DISABLE_TOKEN_INSTANCE_REALTIME_FETCHER='true' -INDEXER_DISABLE_TOKEN_INSTANCE_SANITIZE_FETCHER='true' -INDEXER_DISABLE_WITHDRAWALS_FETCHER='true' - -INDEXER_CATCHUP_BLOCKS_BATCH_SIZE=5 -INDEXER_COIN_BALANCES_BATCH_SIZE=1 -INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE=1 -INDEXER_BLOCK_REWARD_BATCH_SIZE=1 -INDEXER_RECEIPTS_BATCH_SIZE=10 -INDEXER_COIN_BALANCES_BATCH_SIZE=1 -INDEXER_TOKEN_BALANCES_BATCH_SIZE=1 - -INDEXER_CATCHUP_BLOCKS_CONCURRENCY=1 -MIGRATION_TOKEN_INSTANCE_OWNER_BATCH_SIZE=1 -MIGRATION_TOKEN_INSTANCE_OWNER_CONCURRENCY=1 -INDEXER_BLOCK_REWARD_CONCURRENCY=1 -INDEXER_RECEIPTS_CONCURRENCY=1 -INDEXER_COIN_BALANCES_CONCURRENCY=1 -INDEXER_TOKEN_CONCURRENCY=1 -INDEXER_TOKEN_BALANCES_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_RETRY_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_REALTIME_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_SANITIZE_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE=1 -INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE=1 -INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE=1 - -INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT=2 -INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT=2 - -DISABLE_MARKET='true' -SOURCIFY_INTEGRATION_ENABLED='false' - -API_V2_ENABLED=true - -DISABLE_CATCHUP_INDEXER='false' -INDEXER_CATCHUP_BLOCKS_BATCH_SIZE=10 -INDEXER_CATCHUP_BLOCKS_CONCURRENCY=10 -ETHEREUM_JSONRPC_HTTP_URL="https://ethereum-sepolia-rpc.publicnode.com" diff --git a/apps/block_scout_web/assets/package-lock.json b/apps/block_scout_web/assets/package-lock.json index c484b41945c8..a227fb5a0d57 100644 --- a/apps/block_scout_web/assets/package-lock.json +++ b/apps/block_scout_web/assets/package-lock.json @@ -102,10 +102,11 @@ } }, "../../../deps/phoenix": { - "version": "0.0.1" + "version": "1.5.14", + "license": "MIT" }, "../../../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.3.4" }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index 80e1689369c5..a9944a0e797a 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -12,6 +12,8 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do alias Explorer.Chain.Cache.{Accounts, BlockNumber} alias Explorer.Chain.Hash alias Indexer.BufferedTask + alias Indexer.TokenBalances + alias Indexer.Transformers.CoinToTokenBalanceTransformer @doc false # credo:disable-for-next-line Credo.Check.Design.DuplicatedCode @@ -97,17 +99,29 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do value_fetched_at = DateTime.utc_now() importable_balances_params = Enum.map(params_list, &Map.put(&1, :value_fetched_at, value_fetched_at)) - json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) - importable_balances_daily_params = balances_daily_params(params_list, json_rpc_named_arguments) - addresses_params = balances_params_to_address_params(importable_balances_params) + transformed_params = CoinToTokenBalanceTransformer.transform_address_coin_balances(importable_balances_params) + + {coin_balance_entries, token_balance_entries} = + Enum.split_with(transformed_params, fn + {:address_token_balance, _} -> false + _ -> true + end) + + token_balance_params = + Enum.map(token_balance_entries, fn {:address_token_balance, token_balance} -> token_balance end) + + current_token_balance_entries = TokenBalances.to_address_current_token_balances(token_balance_params) + Chain.import(%{ addresses: %{params: addresses_params, with: :balance_changeset}, - address_coin_balances: %{params: importable_balances_params}, + address_coin_balances: %{params: coin_balance_entries}, address_coin_balances_daily: %{params: importable_balances_daily_params}, + address_token_balances: %{params: token_balance_params}, + address_current_token_balances: %{params: current_token_balance_entries}, broadcast: broadcast_type }) end @@ -205,7 +219,7 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do end) end - defp balances_daily_params(params_list, json_rpc_named_arguments) do + def balances_daily_params(params_list, json_rpc_named_arguments) do block_timestamp_map = block_timestamp_map(params_list, json_rpc_named_arguments) params_list diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index 8b2692707892..719c57bfffa1 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -25,6 +25,8 @@ defmodule Indexer.Fetcher.TokenBalance do alias Indexer.{BufferedTask, TokenBalances, Tracer} alias Indexer.Fetcher.TokenBalance.Supervisor, as: TokenBalanceSupervisor + import Indexer.Fetcher.CoinBalance.Helper, only: [balances_daily_params: 2] + @behaviour BufferedTask @default_max_batch_size 100 @@ -210,12 +212,31 @@ defmodule Indexer.Fetcher.TokenBalance do addresses_params = format_and_filter_address_params(token_balances_params) formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) + transformed_params = + Indexer.Transformers.TokenToCoinBalanceTransformer.transform_address_token_balances( + formatted_token_balances_params + ) + + {token_balance_entries, coin_balance_entries} = + Enum.split_with(transformed_params, fn + {:address_coin_balance, _} -> false + _ -> true + end) + + current_token_balances = TokenBalances.to_address_current_token_balances(token_balance_entries) + + coin_balance_params = + Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end) + + json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) + daily_coin_balance_entries = balances_daily_params(coin_balance_params, json_rpc_named_arguments) + import_params = %{ - addresses: %{params: addresses_params}, - address_token_balances: %{params: formatted_token_balances_params}, - address_current_token_balances: %{ - params: TokenBalances.to_address_current_token_balances(formatted_token_balances_params) - }, + addresses: %{params: addresses_params, with: :balance_changeset}, + address_token_balances: %{params: token_balance_entries}, + address_current_token_balances: %{params: current_token_balances}, + address_coin_balances: %{params: coin_balance_params}, + address_coin_balances_daily: %{params: daily_coin_balance_entries}, timeout: @timeout } @@ -234,8 +255,12 @@ defmodule Indexer.Fetcher.TokenBalance do defp format_and_filter_address_params(token_balances_params) do token_balances_params - |> Enum.map(&%{hash: &1.address_hash}) - |> Enum.uniq() + |> Enum.group_by(& &1.address_hash) + |> Map.values() + |> Stream.map(&Enum.max_by(&1, fn %{block_number: block_number} -> block_number end)) + |> Enum.map(fn %{address_hash: address_hash, block_number: block_number, value: value} -> + %{hash: address_hash, fetched_coin_balance_block_number: block_number, fetched_coin_balance: value} + end) end defp format_and_filter_token_balance_params(token_balances_params) do diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex new file mode 100644 index 000000000000..8894ae19d34f --- /dev/null +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex @@ -0,0 +1,27 @@ +defmodule Indexer.Transformers.CoinToTokenBalanceTransformer do + @doc """ + Transforms changes to `address_coin_balances` into changes for `address_token_balances` + if the native token address is set. + """ + def transform_address_coin_balances(params) do + native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") + + Enum.flat_map(params, fn coin_balance -> + if native_token_address do + token_balance = %{ + token_contract_address_hash: native_token_address, + address_hash: coin_balance[:address_hash], + block_number: coin_balance[:block_number], + value: coin_balance[:value], + token_type: "ERC-20", + token_id: 0, + value_fetched_at: coin_balance[:value_fetched_at] + } + + [coin_balance, {:address_token_balance, token_balance}] + else + [coin_balance] + end + end) + end +end diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex new file mode 100644 index 000000000000..37e6568d8107 --- /dev/null +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -0,0 +1,24 @@ +defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do + @doc """ + Transforms changes to `address_token_balances` into changes for `address_coin_balances` + if the token contract address hash matches the hardcoded value. + """ + def transform_address_token_balances(params) do + native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") + + Enum.flat_map(params, fn token_balance -> + if token_balance[:token_contract_address_hash] == native_token_address do + coin_balance = %{ + address_hash: token_balance[:address_hash], + value: token_balance[:value], + block_number: token_balance[:block_number], + value_fetched_at: token_balance[:value_fetched_at] + } + + [token_balance, {:address_coin_balance, coin_balance}] + else + [token_balance] + end + end) + end +end diff --git a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs new file mode 100644 index 000000000000..b1ed6ae98974 --- /dev/null +++ b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs @@ -0,0 +1,314 @@ +defmodule Indexer.XRPLEVM.IntegrationTest do + use EthereumJSONRPC.Case, async: false + use Explorer.DataCase + + import EthereumJSONRPC, only: [integer_to_quantity: 1] + import Mox + + alias Explorer.Chain.{Address, Wei} + + alias Explorer.Repo + + @moduletag :capture_log + + setup :verify_on_exit! + setup :set_mox_global + + setup do + start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) + + initial_config = Application.get_env(:explorer, Explorer.Chain.Cache.BlockNumber) + Application.put_env(:explorer, Explorer.Chain.Cache.BlockNumber, enabled: true) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.BlockNumber, initial_config) + end) + + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + address = insert(:address) + other_address = insert(:address) + block_number = block_number() + block_quantity = integer_to_quantity(block_number) + res = eth_block_number_fake_response(block_quantity) + + {:ok, + native_token_address: native_token_address, + address: address, + other_address: other_address, + block_number: block_number, + block_quantity: block_quantity, + res: res} + end + + defp eth_block_number_fake_response(block_quantity) do + %{ + id: 0, + jsonrpc: "2.0", + result: %{ + "author" => "0x0000000000000000000000000000000000000000", + "difficulty" => "0x20000", + "extraData" => "0x", + "gasLimit" => "0x663be0", + "gasUsed" => "0x0", + "hash" => "0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f", + "logsBloom" => "...", + "miner" => "0x0000000000000000000000000000000000000000", + "number" => block_quantity, + "parentHash" => "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot" => "...", + "sealFields" => ["0x80", "..."], + "sha3Uncles" => "...", + "signature" => "...", + "size" => "0x215", + "stateRoot" => "...", + "step" => "0", + "timestamp" => "0x0", + "totalDifficulty" => "0x20000", + "transactions" => [], + "transactionsRoot" => "...", + "uncles" => [] + } + } + end + + describe "import_token_balances/1 (token→coin)" do + test "updating native token balance updates coin balance", %{ + native_token_address: native_token_address, + address: address, + block_number: block_number, + block_quantity: block_quantity, + res: res + } do + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], + _options -> + {:ok, [res]} + end) + + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + token_contract_address_hash: native_token_address, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(42_000_000) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash) + |> Repo.one() + + assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} + + addr = Repo.get!(Address, address.hash) + assert addr.fetched_coin_balance == %Wei{value: Decimal.new(42_000_000)} + assert addr.fetched_coin_balance_block_number == block_number + + daily = + Explorer.Chain.Address.CoinBalanceDaily + |> where([d], d.address_hash == ^address.hash and d.value == ^Decimal.new(42_000_000)) + |> Repo.one() + + assert daily.day == ~D[1970-01-01] + end + + test "importing non-native token balance does not update coin balance", %{ + address: address, + other_address: other_address, + block_number: block_number + } do + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + token_contract_address_hash: other_address.hash, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert is_nil(coin_balance) + end + end + + describe "import_fetched_balances/2 (coin→token)" do + test "importing native coin balance also creates native token balance", %{ + native_token_address: native_token_address, + address: address, + other_address: other_address, + block_number: block_number, + block_quantity: block_quantity, + res: res + } do + coin_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(123_456_789), + value_fetched_at: DateTime.utc_now() + } + ] + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], + _options -> + {:ok, [res]} + end) + + result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) + assert match?({:ok, _}, result) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} + + other_coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^other_address.hash) + |> Repo.one() + + assert is_nil(other_coin_balance) + + token_balance = + Explorer.Chain.Address.TokenBalance + |> where( + [tb], + tb.address_hash == ^address.hash and tb.block_number == ^block_number and + tb.token_contract_address_hash == ^native_token_address + ) + |> Repo.one() + + assert token_balance.value == Decimal.new(123_456_789) + assert token_balance.token_type == "ERC-20" + + current = + Explorer.Chain.Address.CurrentTokenBalance + |> where( + [ctb], + ctb.address_hash == ^address.hash and + ctb.token_contract_address_hash == ^native_token_address + ) + |> Repo.one() + + assert current.value == Decimal.new(123_456_789) + + addr = Repo.get!(Address, address.hash) + assert addr.fetched_coin_balance == %Wei{value: Decimal.new(123_456_789)} + assert addr.fetched_coin_balance_block_number == block_number + end + + test "latest block snapshot wins", %{ + native_token_address: native_token_address, + address: address, + block_number: block_number + } do + b1 = block_number + b2 = block_number + 1 + q1 = integer_to_quantity(b1) + q2 = integer_to_quantity(b2) + res1 = eth_block_number_fake_response(q1) + res2 = eth_block_number_fake_response(q2) + + EthereumJSONRPC.Mox + |> stub(:json_rpc, fn + [%{method: "eth_getBlockByNumber", params: [^q1, true]}], _opts -> {:ok, [res1]} + [%{method: "eth_getBlockByNumber", params: [^q2, true]}], _opts -> {:ok, [res2]} + end) + + params = [ + %{ + address_hash: to_string(address.hash), + block_number: b1, + value: Decimal.new(100), + value_fetched_at: DateTime.utc_now() + }, + %{ + address_hash: to_string(address.hash), + block_number: b2, + value: Decimal.new(200), + value_fetched_at: DateTime.utc_now() + } + ] + + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) + + current = + Explorer.Chain.Address.CurrentTokenBalance + |> where([ctb], ctb.address_hash == ^address.hash and ctb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert current.value == Decimal.new(200) + end + + test "idempotent coin import doesn't duplicate rows", %{ + address: address, + block_number: block_number + } do + EthereumJSONRPC.Mox + |> stub(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [block_quantity, true] + } + ], + _options -> + {:ok, [eth_block_number_fake_response(block_quantity)]} + end) + + params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(50), + value_fetched_at: DateTime.utc_now() + } + ] + + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) + + assert Repo.aggregate(Explorer.Chain.Address.CoinBalance, :count, :block_number) == 1 + assert Repo.aggregate(Explorer.Chain.Address.CoinBalance, :count, :address_hash) == 1 + end + end +end diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 491adf4a7ad4..b0656a2ebec2 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -38,7 +38,7 @@ ETHEREUM_JSONRPC_TRACE_URL=http://host.docker.internal:8545/ SECRET_KEY_BASE=56NtB48ear7+wMSf0IQuWDAAazhpb31qyc7GiyspBP2vh7t5zlCsF5QDv76chXeN # CHECK_ORIGIN= PORT=4000 -COIN_NAME= +COIN_NAME=XRP # METADATA_CONTRACT= # VALIDATORS_CONTRACT= # KEYS_MANAGER_CONTRACT= @@ -47,7 +47,9 @@ COIN_NAME= # CHAIN_SPEC_PATH= # CHAIN_SPEC_PROCESSING_DELAY= # SUPPLY_MODULE= -COIN= +COIN=XRP +NATIVE_TOKEN_ADDRESS=0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE +NATIVE_TOKEN_ID=0xba5a21ca88ef6bba2bfff5088994f90e1077e2a1cc3dcc38bd261f00fce2824f DISABLE_MARKET=true # MARKET_NATIVE_COIN_SOURCE= # MARKET_SECONDARY_COIN_SOURCE= @@ -343,8 +345,8 @@ API_V1_WRITE_METHODS_DISABLED=false # INDEXER_ROOTSTOCK_DATA_FETCHER_BATCH_SIZE= # INDEXER_ROOTSTOCK_DATA_FETCHER_CONCURRENCY= # INDEXER_ROOTSTOCK_DATA_FETCHER_DB_BATCH_SIZE= -# INDEXER_BEACON_RPC_URL=http://localhost:5052 -# INDEXER_DISABLE_BEACON_BLOB_FETCHER= +INDEXER_BEACON_RPC_URL=https://rpc.testnet.xrplevm.org +INDEXER_DISABLE_BEACON_BLOB_FETCHER=true # INDEXER_BEACON_BLOB_FETCHER_SLOT_DURATION=12 # INDEXER_BEACON_BLOB_FETCHER_REFERENCE_SLOT=8000000 # INDEXER_BEACON_BLOB_FETCHER_REFERENCE_TIMESTAMP=1702824023