Merge branch 'main' into feat/mtg-1164

.dockerignore

@@ -4,4 +4,5 @@ target
 db-data
 node-modules
 ledger
-.anchor
+tmp
+.anchor

.env.example

@@ -1,122 +1,55 @@
-# Required by Postgre container
-POSTGRE_DB_PATH="postgre/db/path"
-
-RUST_BACKTRACE=1
-# Ingester instance config
-INGESTER_LOG_LEVEL=info
-
-INGESTER_DATABASE_CONFIG='{max_postgres_connections=10, url="postgres://user:[email protected]:5432/database"}'
-INGESTER_TCP_CONFIG='{receiver_addr="localhost:2000", receiver_reconnect_interval=5, snapshot_receiver_addr="localhost:5000"}'
-INGESTER_REDIS_MESSENGER_CONFIG='{messenger_type="Redis", connection_config={redis_connection_str="redis://:pass@localhost:6379"}}'
-INGESTER_MESSAGE_SOURCE=Redis #TCP or Redis
-
-INGESTER_ACCOUNTS_BUFFER_SIZE=250
-INGESTER_ACCOUNTS_PARSING_WORKERS=20
-INGESTER_TRANSACTIONS_PARSING_WORKERS=20
-
-INGESTER_SNAPSHOT_PARSING_WORKERS=1
-INGESTER_SNAPSHOT_PARSING_BATCH_SIZE=250
-
-INGESTER_GAPFILLER_PEER_ADDR="0.0.0.0"
-INGESTER_METRICS_PORT=9091
-INGESTER_SERVER_PORT=9092
-INGESTER_PEER_GRPC_PORT=9099
-
-INGESTER_ROCKS_DB_PATH_CONTAINER="/usr/src/rocksdb-data"
-INGESTER_ROCKS_DB_PATH="path/to/rocks/on/disk"
-
-INGESTER_ARCHIVES_DIR="path/to/rocks/backup/archives"
-INGESTER_ROCKS_BACKUP_ARCHIVES_DIR="path/to/rocks/backup/archives"
-INGESTER_ROCKS_BACKUP_DIR="path/to/rocks/backup/"
-
-INGESTER_BACKFILL_RPC_ADDRESS='https://rpc:port'
-INGESTER_RPC_HOST='https://rpc:port'
-
-INGESTER_BACKFILLER_SOURCE_MODE=RPC #RPC or Bigtable
-INGESTER_BIG_TABLE_CONFIG='{creds="/usr/src/app/creds.json", timeout=1000}'
-
-INGESTER_RUN_SEQUENCE_CONSISTENT_CHECKER=false # experimental, enable only for testing purposes
-# Optional, required only if it needs to run fork cleaner, default is false. Unstable as it removes forked items, but also removes some valid leafs. Recommended to use only! for testing purposes.
-INGESTER_RUN_FORK_CLEANER=false
-INGESTER_RUN_BUBBLEGUM_BACKFILLER=true
-
-INGESTER_BACKFILLER_MODE=PersistAndIngest # The only available option, the variable will be removed
-INGESTER_SLOT_UNTIL=0
-INGESTER_SLOT_START_FROM=0
-INGESTER_WORKERS_COUNT=100
-INGESTER_CHUNK_SIZE=20
-INGESTER_PERMITTED_TASKS=1
-INGESTER_WAIT_PERIOD_SEC=30
-INGESTER_SHOULD_REINGEST=false
-
-INGESTER_PEER_GRPC_MAX_GAP_SLOTS=1000000
-
-INGESTER_RUN_PROFILING=false
-INGESTER_PROFILING_FILE_PATH_CONTAINER="/usr/src/profiling"
-INGESTER_PROFILING_FILE_PATH="/path/to/profiling"
+RPC_HOST='https://mainnet-aura.metaplex.com/{personal_rpc_key}'
 
-INGESTER_FILE_STORAGE_PATH_CONTAINER="/usr/src/app/file_storage"
-INGESTER_FILE_STORAGE_PATH="path/to/file/storage"
-INGESTER_MIGRATION_STORAGE_PATH=/path/to/migration_storage # requires explanation
+#Postgres
+PG_DATABASE_URL='postgres://solana:solana@localhost:5432/aura_db'
 
-INGESTER_ROCKS_FLUSH_BEFORE_BACKUP=false
-INGESTER_ROCKS_INTERVAL_IN_SECONDS=3600
-INGESTER_ROCKS_SYNC_INTERVAL_SECONDS=2
+#Redis
+REDIS_CONNECTION_CONFIG='{"redis_connection_str":"redis://127.0.0.1:6379/0"}'
 
-INGESTER_SYNCHRONIZER_DUMP_PATH="/path/to/dump"
-INGESTER_DISABLE_SYNCHRONIZER=true
-INGESTER_SKIP_CHECK_TREE_GAPS=true
+#RocksDB
+ROCKS_DB_PATH="/usr/src/rocksdb-data"
+ROCKS_DB_PATH_CONTAINER="/usr/src/rocksdb-data"
+ROCKS_DB_SECONDARY_PATH_CONTAINER="path/to/rocks/secondary/db"
 # path to the slots data, required for the backfiller to work
-INGESTER_SLOTS_DB_PATH=/path/to/slots-data
-INGESTER_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots # should be removed
+ROCKS_SLOTS_DB_PATH=/path/to/slots-data
+ROCKS_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots
+ROCKS_ARCHIVES_DIR="path/to/rocks/backup/archives"
+ROCKS_BACKUP_ARCHIVES_DIR="path/to/rocks/backup/archives"
+ROCKS_MIGRATION_STORAGE_PATH=/path/to/migration_storage
 
-# a common log level for all instances, will be overridden by specific log levels, requires refactoring
-RUST_LOG=info
-# API instance config
-API_LOG_LEVEL=info
+#Backfiller
+BACKFILLER_SOURCE_MODE=bigtable
+BIG_TABLE_CONFIG='{creds="/usr/src/app/creds.json", timeout=1000}'
 
-API_DATABASE_CONFIG='{max_postgres_connections=250, url="postgres://user:[email protected]:5432/database"}'
-
-API_ROCKS_DB_PATH_CONTAINER="/usr/src/rocksdb-data"
-API_ROCKS_DB_SECONDARY_PATH_CONTAINER="path/to/rocks/secondary/db"
-API_ARCHIVES_DIR="path/to/rocks/backup/archives"
-
-API_PEER_GRPC_PORT=8991
+# Metrics port. Start HTTP server to report metrics if port exist.
 API_METRICS_PORT=8985
-API_SERVER_PORT=8990
-
-API_RPC_HOST='https://rpc:port'
-
-API_ROCKS_SYNC_INTERVAL_SECONDS=2
-API_FILE_STORAGE_PATH_CONTAINER="/usr/src/app/file_storage"
-API_FILE_STORAGE_PATH="path/to/file/storage"
-
-API_PEER_GRPC_MAX_GAP_SLOTS=1000000
-API_JSON_MIDDLEWARE_CONFIG='{is_enabled=true, max_urls_to_parse=10}'
-
-API_CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD=1000000
-API_CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD=500
-
-# if set to true API will not check if tree where user requests assets from has any gaps
-API_SKIP_CHECK_TREE_GAPS=true
-
-# Synchronizer instance config
-SYNCHRONIZER_LOG_LEVEL=info
+INGESTER_METRICS_PORT=9091
+MIGRATOR_METRICS_PORT=5091
+SYNCHRONIZER_METRICS_PORT=6091
 
-SYNCHRONIZER_DATABASE_CONFIG='{max_postgres_connections=100, url="postgres://user:[email protected]:5432/database"}'
-SYNCHRONIZER_ROCKS_DB_PATH_CONTAINER="/usr/src/rocksdb-data"
-SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER="path/to/rocks/secondary/db" # should be removed
+# API server port (if API is enabled)
+INGESTER_SERVER_PORT=9092
+API_SERVER_PORT=8990
 
-SYNCHRONIZER_METRICS_PORT=6091
+# GRPC Server port
+PEER_GRPC_PORT=9099
 
-SYNCHRONIZER_DUMP_PATH="/path/to/migration_data"
+# Docker
+# Required by Postgre container
+POSTGRE_DB_PATH="postgre/db/path"
+ROCKS_BACKUP_DIR="path/to/rocks/backup/"
+FILE_STORAGE_PATH="path/to/file/storage"
+FILE_STORAGE_PATH_CONTAINER="/usr/src/app/file_storage"
+ROCKS_DUMP_PATH="/path/to/dump"
 
-SYNCHRONIZER_DUMP_SYNCHRONIZER_BATCH_SIZE=10000
-# threshold on the number of updates not being synchronized for the synchronizer to dump-load on start 
-# 150M - that's a rough threshold after which the synchronizer will likely complete a full dymp-load cycle faster then doing an incremental sync
-SYNCHRONIZER_DUMP_SYNC_THRESHOLD=150000000
+#Profiling (optional)
+PROFILING_FILE_PATH_CONTAINER="/usr/src/profiling"
+PROFILING_FILE_PATH="/path/to/profiling"
 
-SYNCHRONIZER_PARALLEL_TASKS=30
+# DEV configuration
+RUST_BACKTRACE=1
+# warn|info|debug
+LOG_LEVEL=info
 
 # Profiling config
 # Optional, required only if it needs to run memory profiling
@@ -126,4 +59,71 @@ MALLOC_CONF="prof:true,prof_leak:true,prof_final:true,prof_active:true,prof_pref
 INTEGRITY_VERIFICATION_TEST_FILE_PATH="./test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_TEST_FILE_PATH_CONTAINER="/test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH="./slots_collect"
-INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH_CONTAINER="/slots_collect"
+INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH_CONTAINER="/slots_collect"
+
+#Configurable app parts that cold be enabled or disabled. (values in the default positions)
+RUN_API=true
+RUN_BACKFILLER=true
+RUN_BUBBLEGUM_BACKFILLER=true
+RUN_GAPFILLER=false
+SHOULD_REINGEST=false
+RUN_PROFILING=false
+RESTORE_ROCKS_DB=false
+ENABLE_ROCKS_MIGRATION=true
+CHECK_PROOFS=false
+SKIP_CHECK_TREE_GAPS=false
+
+#Changes (todo Remove after review/migration)
+
+# API_RPC_HOST  INGESTER_RPC_HOST -> RPC_HOST
+#INGESTER_ROCKS_DB_PATH -> ROCKS_DB_PATH
+#INGESTER_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
+#INGESTER_SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
+#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
+#INGESTER_FILE_STORAGE_PATH -> FILE_STORAGE_PATH
+#INGESTER_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
+# INGESTER_PROFILING_FILE_PATH -> PROFILING_FILE_PATH
+# INGESTER_PROFILING_FILE_PATH_CONTAINER -> PROFILING_FILE_PATH_CONTAINER
+#  INGESTER_MIGRATION_STORAGE_PATH -> ROCKS_MIGRATION_STORAGE_PATH
+#
+# INGESTER_ROCKS_BACKUP_ARCHIVES_DIR -> ROCKS_BACKUP_ARCHIVES_DIR
+# INGESTER_ROCKS_BACKUP_DIR ->  ROCKS_BACKUP_DIR
+#
+#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
+#INGESTER_SLOTS_DB_PATH -> ROCKS_SLOTS_DB_PATH
+#INGESTER_SECONDARY_SLOTS_DB_PATH -> ROCKS_SECONDARY_SLOTS_DB_PATH
+
+#API_DATABASE_CONFIG ->  PG_MAX_DB_CONNECTIONS  and  PG_DATABASE_URL
+
+#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
+#SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH_CONTAINER
+#SYNCHRONIZER_DUMP_SYNCHRONIZER_BATCH_SIZE -> DUMP_SYNCHRONIZER_BATCH_SIZE
+#SYNCHRONIZER_DUMP_SYNC_THRESHOLD -> DUMP_SYNC_THRESHOLD
+
+#API_ARCHIVES_DIR -> rocks_archives_dir
+#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH_CONTAINER
+#API_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH_CONTAINER
+#API_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
+#API_JSON_MIDDLEWARE_CONFIG -> JSON_MIDDLEWARE_CONFIG
+#API_CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD - > CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD
+#API_CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD -> CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD
+#
+#API_SKIP_CHECK_TREE_GAPS -> SKIP_CHECK_TREE_GAPS
+# INGESTER_REDIS_MESSENGER_CONFIG -> REDIS_CONNECTION_CONFIG and note the differnce (less complex type)
+# INGESTER_BACKFILLER_SOURCE_MODE -> BACKFILLER_SOURCE_MODE
+# INGESTER_BIG_TABLE_CONFIG -> BIG_TABLE_CONFIG
+# both API_PEER_GRPC_PORT and INGESTER_PEER_GRPC_PORT were replaced with a single PEER_GRPC_PORT - only one will actually work
+# API_PEER_GRPC_MAX_GAP_SLOTS and INGESTER_PEER_GRPC_MAX_GAP_SLOTS -> PEER_GRPC_MAX_GAP_SLOTS (optional)
+# INGESTER_ROCKS_SYNC_INTERVAL_SECONDS and API_ROCKS_SYNC_INTERVAL_SECONDS -> ROCKS_SYNC_INTERVAL_SECONDS (optional)
+# INGESTER_GAPFILLER_PEER_ADDR -> GAPFILLER_PEER_ADDR (optional, only if run_gapfiller is set)
+# INGESTER_ACCOUNTS_BUFFER_SIZE -> ACCOUNT_PROCESSOR_BUFFER_SIZE (has default value, may be skipped)
+# INGESTER_ACCOUNTS_PARSING_WORKERS -> REDIS_ACCOUNTS_PARSING_WORKERS
+# INGESTER_ROCKS_FLUSH_BEFORE_BACKUP -> ROCKS_FLUSH_BEFORE_BACKUP
+# INGESTER_ROCKS_INTERVAL_IN_SECONDS -> ROCKS_INTERVAL_IN_SECONDS
+# INGESTER_TRANSACTIONS_PARSING_WORKERS -> REDIS_TRANSACTIONS_PARSING_WORKERS
+# INGESTER_SHOULD_REINGEST -> SHOULD_REINGEST
+# INGESTER_RUN_SEQUENCE_CONSISTENT_CHECKER -> RUN_SEQUENCE_CONSISTENT_CHECKER
+# INGESTER_RUN_PROFILING -> RUN_PROFILING
+# INGESTER_RUN_BUBBLEGUM_BACKFILLER -> RUN_BUBBLEGUM_BACKFILLER
+# Removed:
+# INGESTER_BACKFILL_RPC_ADDRESS, INGESTER_BACKFILLER_MODE, INGESTER_MESSAGE_SOURCE (redis is used as an only option as of now), INGESTER_DISABLE_SYNCHRONIZER (synchronizer is no longer part of the ingester), INGESTER_CHUNK_SIZE, INGESTER_PERMITTED_TASKS, INGESTER_TCP_CONFIG, INGESTER_WORKERS_COUNT, INGESTER_WAIT_PERIOD_SEC, INGESTER_SNAPSHOT_PARSING_WORKERS, INGESTER_SNAPSHOT_PARSING_BATCH_SIZE, INGESTER_SLOT_UNTIL, INGESTER_SLOT_START_FROM, INGESTER_RUN_FORK_CLEANER, INGESTER_RUN_DUMP_SYNCHRONIZE_ON_START