Single schema for all evm chains

src/__tests__/qrid.test.ts

@@ -25,6 +25,7 @@ test('Old QR ID works', () => {
         transactionIndex: 81,
         type: 2,
         value: new BigNumber('0'),
+        chainId: '1',
     };
 
     const rawTransaction: EVMTransaction = {
@@ -67,6 +68,7 @@ test('New ZID works', () => {
         transactionIndex: 81,
         type: 2,
         value: new BigNumber('0'),
+        chainId: '1',
     };
 
     const rawTransaction: EVMTransaction = {

src/entities/block.ts

@@ -21,4 +21,6 @@ export class Block {
     public gasUsed!: number;
     @Column({ name: 'parent_hash', type: 'varchar' })
     public parentHash!: string;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/current_epoch_info.ts

@@ -29,4 +29,6 @@ export class CurrentEpochInfo {
     // Relevant ZRX staked for this epoch
     @Column({ name: 'zrx_staked', type: 'numeric', transformer: bigNumberTransformer })
     public zrxStaked!: BigNumber;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/event.ts

@@ -24,4 +24,6 @@ export abstract class Event {
     // The block number where the event occurred.
     @Column({ name: 'block_number', type: 'bigint', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/event_backfill.ts

@@ -9,4 +9,6 @@ export class EventBackfill {
     // depth of the block
     @PrimaryColumn({ name: 'block_number', type: 'bigint', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/last_block_processed.ts

@@ -29,4 +29,6 @@ export class LastBlockProcessed {
 
     @Column({ name: 'block_hash', type: 'varchar' })
     public blockHash!: string | null;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/staking_pool_earned_rewards_in_epoch_event.ts

@@ -11,4 +11,6 @@ export class StakingPoolEarnedRewardsInEpochEvent extends Event {
     // The ID of the pool.
     @Column({ name: 'pool_id', type: 'varchar' })
     public poolId!: string;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/staking_pool_metadata.ts

@@ -24,4 +24,6 @@ export class StakingPoolMetadata {
     // whether or not the info has been verified by the 0x core team
     @Column({ name: 'verified', type: 'boolean' })
     public verified!: boolean;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/staking_proxy_deployment.ts

@@ -23,4 +23,6 @@ export class StakingProxyDeployment {
     // The timestamp of the block where the transaction occurred
     @Column({ name: 'block_timestamp', type: 'bigint', transformer: numberToBigIntTransformer })
     public blockTimestamp!: number;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/token_metadata.ts

@@ -16,4 +16,6 @@ export class TokenMetadata {
     public name!: string | null;
     @Column({ name: 'decimals', type: 'int', transformer: numberToBigIntTransformer })
     public decimals!: BigNumber | null;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/transaction.ts

@@ -59,4 +59,6 @@ export class Transaction {
     // priority fee in wei
     @Column({ name: 'max_priority_fee_per_gas', type: 'numeric', transformer: bigNumberTransformer })
     public maxPriorityFeePerGas!: BigNumber | null;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/transaction_log.ts

@@ -19,4 +19,6 @@ export class TransactionLogs {
     // depth of the block
     @Column({ name: 'block_number', type: 'bigint', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/entities/transaction_receipt.ts

@@ -32,4 +32,6 @@ export class TransactionReceipt {
     // Amount of gas consumed by the tx in L1 (only filled when this amount is not included in the L2 gas, (like in OP stack chains)
     @Column({ name: 'gas_fees_l1', type: 'numeric', transformer: bigNumberTransformer })
     public gasFeesL1!: BigNumber | null;
+    @Column({ name: 'chain_id', type: 'varchar' })
+    public chainId!: string;
 }

src/parsers/events/parse_event.ts

@@ -1,5 +1,6 @@
 import { Event } from '../../entities';
 import { LogEntry, RawLogEntry } from 'ethereum-types';
+import { CHAIN_ID } from '../../config';
 
 export function parseEvent(eventLog: LogEntry | RawLogEntry, eventEntity: Event) {
     eventEntity.observedTimestamp = new Date().getTime();
@@ -9,4 +10,5 @@ export function parseEvent(eventLog: LogEntry | RawLogEntry, eventEntity: Event)
     eventEntity.logIndex = eventLog.logIndex as number;
     eventEntity.blockHash = (eventLog.blockHash as string).toLowerCase();
     eventEntity.blockNumber = eventLog.blockNumber as number;
+    eventEntity.chainId = CHAIN_ID.toString();
 }

src/parsers/staking-pool-registry/index.ts

@@ -1,3 +1,4 @@
+import { CHAIN_ID } from '../../config';
 import { BASE_GITHUB_LOGO_URL } from '../../config';
 import { MetadataResponse, PoolMetadataResponse, Pools } from '../../data_sources/staking-pool-registry';
 import { StakingPoolMetadata } from '../../entities';
@@ -30,6 +31,7 @@ function parseStakingPoolMetadata(poolMetadataResponse: PoolMetadataResponse, po
     stakingPoolMetadata.logoUrl =
         poolMetadataResponse.logo_img === undefined ? null : baseGithubLogoUrl.concat(poolMetadataResponse.logo_img);
     stakingPoolMetadata.verified = poolMetadataResponse.verified;
+    stakingPoolMetadata.chainId = CHAIN_ID.toString();
 
     return stakingPoolMetadata;
 }

src/parsers/web3/parse_web3_objects.ts

@@ -85,6 +85,7 @@ export function parseTransaction(rawTx: EVMTransaction): Transaction {
     transaction.maxFeePerGas = rawTx.maxFeePerGas === undefined ? null : new BigNumber(rawTx.maxFeePerGas);
     transaction.maxPriorityFeePerGas =
         rawTx.maxPriorityFeePerGas === undefined ? null : new BigNumber(rawTx.maxPriorityFeePerGas);
+    transaction.chainId = CHAIN_ID.toString();
 
     if (
         transaction.input.includes(SETTLER_EXECUTE_SELECTOR) ||
@@ -165,6 +166,7 @@ export function parseTransactionReceipt(rawReceipt: RawReceipt): TransactionRece
     transactionReceipt.toAddress = rawReceipt.to;
     transactionReceipt.gasUsed = new BigNumber(rawReceipt.gasUsed);
     transactionReceipt.gasFeesL1 = rawReceipt.gasFeesL1 === undefined ? null : new BigNumber(rawReceipt.gasFeesL1);
+    transactionReceipt.chainId = CHAIN_ID.toString();
 
     return transactionReceipt;
 }
@@ -181,6 +183,7 @@ export function parseTransactionLogs(rawReceipt: RawReceipt): TransactionLogs {
     transactionLogs.logs = JSON.stringify(rawReceipt.logs);
     transactionLogs.blockHash = rawReceipt.blockHash === null ? '' : rawReceipt.blockHash;
     transactionLogs.blockNumber = rawReceipt.blockNumber === null ? 0 : rawReceipt.blockNumber;
+    transactionLogs.chainId = CHAIN_ID.toString();
 
     return transactionLogs;
 }
@@ -199,6 +202,7 @@ export function parseBlock(rawBlock: BlockWithoutTransactionData): Block {
     parsedBlock.baseFeePerGas = rawBlock.baseFeePerGas;
     parsedBlock.gasUsed = rawBlock.gasUsed;
     parsedBlock.parentHash = rawBlock.parentHash;
+    parsedBlock.chainId = CHAIN_ID.toString();
 
     return parsedBlock;
 }

src/scripts/backfill_events.ts

@@ -1,4 +1,4 @@
-import { EVM_RPC_URL } from '../config';
+import { EVM_RPC_URL, CHAIN_ID } from '../config';
 import { Web3Source } from '../data_sources/events/web3';
 import { EventBackfill } from '../entities';
 import { eventScrperProps, EventScraperProps, CommonEventParams } from '../events';
@@ -78,6 +78,7 @@ export class EventsBackfillScraper {
                                         .from(EventBackfill)
                                         .where('blockNumber >= :startBlockNumber', { startBlockNumber })
                                         .andWhere('blockNumber <= :endBlockNumber', { endBlockNumber })
+                                        .andWhere('chainId = :chainId', { "chainId": CHAIN_ID })
                                         .execute();
                                 }
 

src/scripts/pull_and_save_backfill_tx.ts

@@ -1,4 +1,4 @@
-import { EVM_RPC_URL, MAX_TX_TO_PULL, SCHEMA } from '../config';
+import { CHAIN_ID, EVM_RPC_URL, MAX_TX_TO_PULL, SCHEMA } from '../config';
 import { Web3Source } from '../data_sources/events/web3';
 import { Transaction, TransactionReceipt } from '../entities';
 import { chunk, logger } from '../utils';
@@ -35,11 +35,11 @@ export class BackfillTxScraper {
             const txData = await getParseTxsAsync(web3Source, txList);
 
             const txHashList = txData.parsedTxs.map((tx) => `'${tx.transactionHash}'`).toString();
-            const txDeleteQuery = `DELETE FROM ${SCHEMA}.transactions WHERE transaction_hash IN (${txHashList})`;
-            const txReceiptDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_receipts WHERE transaction_hash IN (${txHashList});`;
-            // const txLogsDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_logs WHERE transaction_hash IN (${txHashList});`;
+            const txDeleteQuery = `DELETE FROM ${SCHEMA}.transactions_${CHAIN_ID} WHERE transaction_hash IN (${txHashList})`;
+            const txReceiptDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_receipts_${CHAIN_ID} WHERE transaction_hash IN (${txHashList});`;
+            // const txLogsDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_logs_${CHAIN_ID} WHERE transaction_hash IN (${txHashList});`;
             const txBacklogQuery = `
-              UPDATE ${SCHEMA}.tx_backfill
+              UPDATE ${SCHEMA}.tx_backfill_${CHAIN_ID}
               SET done = true
               WHERE transaction_hash IN (${txHashList})`;
 

src/scripts/pull_and_save_block_events.ts

@@ -1,5 +1,6 @@
 import {
     BLOCKS_REORG_CHECK_INCREMENT,
+    CHAIN_ID,
     CHAIN_NAME,
     EVM_RPC_URL,
     MAX_BLOCKS_REORG,
@@ -208,7 +209,7 @@ async function saveFullBlocks(connection: Connection, eventTables: string[], par
             blockRanges.forEach(async (blockRange) => {
                 deletePromises.push(
                     queryRunner.manager.query(
-                        `DELETE FROM ${SCHEMA}.${tableName}
+                        `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID}
                      WHERE
                        block_number >= ${blockRange.start} AND
                        block_number <= ${blockRange.end}`,
@@ -475,7 +476,7 @@ export class BlockEventsScraper {
                     const queryRunner = connection.createQueryRunner();
                     await queryRunner.connect();
                     await queryRunner.manager.query(
-                        `DELETE FROM ${SCHEMA}.blocks
+                        `DELETE FROM ${SCHEMA}.blocks_${CHAIN_ID}
                          WHERE block_number > ${testBlockNumber}`,
                     );
                     queryRunner.release();

src/scripts/pull_and_save_tokens_backfill.ts

@@ -1,4 +1,4 @@
-import { EVM_RPC_URL, MAX_BLOCKS_TO_SEARCH, SCHEMA } from '../config';
+import { CHAIN_ID, EVM_RPC_URL, MAX_BLOCKS_TO_SEARCH, SCHEMA } from '../config';
 import { Web3Source } from '../data_sources/events/web3';
 import { logger } from '../utils/logger';
 import { SCRIPT_RUN_DURATION } from '../utils/metrics';
@@ -30,7 +30,7 @@ export class TokensFromBackfill {
             const savedTokenCount = await getParseSaveTokensAsync(connection, producer, web3Source, tokens);
 
             const tokenList = tokens.map((token: string) => `'${token}'`).toString();
-            const tokenBackfillQuery = `DELETE FROM ${SCHEMA}.tokens_backfill WHERE address IN (${tokenList});`;
+            const tokenBackfillQuery = `DELETE FROM ${SCHEMA}.tokens_backfill_${CHAIN_ID} WHERE address IN (${tokenList});`;
             const queryRunner = connection.createQueryRunner();
             await queryRunner.connect();
 

src/scripts/utils/event_abi_utils.ts

@@ -1,4 +1,4 @@
-import { CHAIN_NAME_LOWER, MAX_BLOCKS_REORG, MAX_BLOCKS_TO_SEARCH, RESCRAPE_BLOCKS, SCHEMA } from '../../config';
+import { CHAIN_ID, CHAIN_NAME_LOWER, MAX_BLOCKS_REORG, MAX_BLOCKS_TO_SEARCH, RESCRAPE_BLOCKS, SCHEMA } from '../../config';
 import { LogPullInfo, Web3Source } from '../../data_sources/events/web3';
 import { Event } from '../../entities';
 import { LastBlockProcessed } from '../../entities';
@@ -267,24 +267,24 @@ export class PullAndSaveEventsByTopic {
         let deleteQuery = '';
         if (tableName === 'erc20_bridge_transfer_events') {
             if (deleteOptions.directFlag && deleteOptions.directProtocol != undefined) {
-                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND direct_protocol IN ('${deleteOptions.directProtocol.join(
+                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND direct_protocol IN ('${deleteOptions.directProtocol.join(
                     "','",
                 )}')`;
             } else if (deleteOptions.directFlag === false) {
-                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND direct_flag = FALSE`;
+                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND direct_flag = FALSE`;
             }
         } else if (tableName === 'native_fills' && deleteOptions.protocolVersion != undefined) {
             if (deleteOptions.protocolVersion === 'v4' && deleteOptions.nativeOrderType != undefined) {
-                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol_version = '${deleteOptions.protocolVersion}' AND native_order_type = '${deleteOptions.nativeOrderType}' `;
+                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol_version = '${deleteOptions.protocolVersion}' AND native_order_type = '${deleteOptions.nativeOrderType}' `;
             } else {
-                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol_version = '${deleteOptions.protocolVersion}'`;
+                deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol_version = '${deleteOptions.protocolVersion}'`;
             }
         } else if (tableName === 'uniswap_v2_pair_created_events') {
-            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol = '${deleteOptions.protocol}'`;
+            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND protocol = '${deleteOptions.protocol}'`;
         } else if (tableName === 'log_transfer_events') {
-            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND "to" = '${deleteOptions.recipient}'`;
+            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock} AND "to" = '${deleteOptions.recipient}'`;
         } else {
-            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock}`;
+            deleteQuery = `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlock} AND block_number <= ${endBlock}`;
         }
 
         await queryRunner.connect();
@@ -408,5 +408,6 @@ export const getLastBlockProcessedEntity = (
     lastBlockProcessed.lastProcessedBlockNumber = endBlockNumber;
     lastBlockProcessed.processedTimestamp = new Date().getTime();
     lastBlockProcessed.blockHash = endBlockHash;
+    lastBlockProcessed.chainId = CHAIN_ID.toString();
     return lastBlockProcessed;
 };

src/scripts/utils/event_utils.ts

@@ -1,4 +1,4 @@
-import { FIRST_SEARCH_BLOCK, MAX_BLOCKS_REORG, MAX_BLOCKS_TO_SEARCH, SCHEMA } from '../../config';
+import { CHAIN_ID, FIRST_SEARCH_BLOCK, MAX_BLOCKS_REORG, MAX_BLOCKS_TO_SEARCH, SCHEMA } from '../../config';
 import { Web3Source } from '../../data_sources/events/web3';
 import { LastBlockProcessed } from '../../entities';
 import { logger } from '../../utils/logger';
@@ -102,7 +102,7 @@ export class PullAndSaveEvents {
         try {
             // delete events scraped prior to the most recent block range
             await queryRunner.manager.query(
-                `DELETE FROM ${SCHEMA}.${tableName} WHERE block_number >= ${startBlockNumber} AND block_number <= ${endBlock}`,
+                `DELETE FROM ${SCHEMA}.${tableName}_${CHAIN_ID} WHERE block_number >= ${startBlockNumber} AND block_number <= ${endBlock}`,
             );
             await queryRunner.manager.save(toSave);
             await queryRunner.manager.save(lastBlockProcessed);