diff --git a/.env.example b/.env.example index bb03ce273..dea05c3c7 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,8 @@ CONSENSUS_CLIENT_URI=http://... EXECUTION_CLIENT_URI=http://... KEYS_API_URI=https://... LIDO_LOCATOR_ADDRESS=0x1... -CSM_MODULE_ADDRESS=0x... +CS_MODULE_ADDRESS=0x... +CURATED_MODULE_ADDRESS=0x... MEMBER_PRIV_KEY=aaa... PINATA_JWT=... STORACHA_AUTH_SECRET=... diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9c38ecd35..b95edc8f8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -14,4 +14,4 @@ Describe how you tested the changes: ## Checklist - [ ] Documentation updated (if required) - [ ] New tests added (if applicable) -- [ ] `CSM_STATE_VERSION` is bumped (if the new version affects data in the cache) +- [ ] `STAKING_MODULE_STATE_VERSION` is bumped (if the new version affects data in the cache) diff --git a/.github/workflows/ci-daily-checks.yml b/.github/workflows/ci-daily-checks.yml index 1af0b9d44..51cb49e6b 100644 --- a/.github/workflows/ci-daily-checks.yml +++ b/.github/workflows/ci-daily-checks.yml @@ -22,7 +22,8 @@ jobs: CONSENSUS_CLIENT_URI: ${{ secrets.CONSENSUS_CLIENT_URI }} KEYS_API_URI: ${{ secrets.KEYS_API_URI }} LIDO_LOCATOR_ADDRESS: ${{ secrets.LIDO_LOCATOR_ADDRESS }} - CSM_MODULE_ADDRESS: ${{ secrets.CSM_MODULE_ADDRESS }} + CS_MODULE_ADDRESS: ${{ secrets.CS_MODULE_ADDRESS }} + CURATED_MODULE_ADDRESS: ${{ secrets.CURATED_MODULE_ADDRESS }} PINATA_JWT: ${{ secrets.PINATA_JWT }} PINATA_DEDICATED_GATEWAY_URL: ${{ secrets.PINATA_DEDICATED_GATEWAY_URL }} PINATA_DEDICATED_GATEWAY_TOKEN: ${{ secrets.PINATA_DEDICATED_GATEWAY_TOKEN}} @@ -39,7 +40,8 @@ jobs: CONSENSUS_CLIENT_URI=${CONSENSUS_CLIENT_URI} KEYS_API_URI=${KEYS_API_URI} LIDO_LOCATOR_ADDRESS=${LIDO_LOCATOR_ADDRESS} - CSM_MODULE_ADDRESS=${CSM_MODULE_ADDRESS} + CS_MODULE_ADDRESS=${CS_MODULE_ADDRESS} + CURATED_MODULE_ADDRESS=${CURATED_MODULE_ADDRESS} GW3_ACCESS_KEY=${GW3_ACCESS_KEY} GW3_SECRET_KEY=${GW3_SECRET_KEY} PINATA_JWT=${PINATA_JWT} diff --git a/.github/workflows/mainnet_fork_tests.yml b/.github/workflows/mainnet_fork_tests.yml index 0727242c7..fcf865703 100644 --- a/.github/workflows/mainnet_fork_tests.yml +++ b/.github/workflows/mainnet_fork_tests.yml @@ -26,38 +26,6 @@ jobs: steps: - uses: actions/checkout@v3 - # TODO: Remove after upgrade to CSM v2 on Mainnet. - - name: Checkout CSM repo - uses: actions/checkout@v4 - with: - repository: 'lidofinance/community-staking-module' - ref: 'develop' - path: 'testruns/community-staking-module' - persist-credentials: false - - - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 - with: - version: ${{ env.FORGE_REV }} - - - name: Install node - uses: actions/setup-node@v4 - with: - node-version-file: "testruns/community-staking-module/.nvmrc" - cache: 'yarn' - cache-dependency-path: "testruns/community-staking-module/yarn.lock" - - - name: Install Just - run: cargo install "just@1.24.0" - - - name: Install dependencies - working-directory: testruns/community-staking-module - run: just deps - - - name: Build contracts - working-directory: testruns/community-staking-module - run: just build - - name: Set up Python 3.12 uses: actions/setup-python@v4 with: @@ -82,7 +50,8 @@ jobs: CONSENSUS_CLIENT_URI: ${{ secrets.CONSENSUS_CLIENT_URI }} KEYS_API_URI: ${{ secrets.KEYS_API_URI }} LIDO_LOCATOR_ADDRESS: "0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb" - CSM_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" + CS_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" + CURATED_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" # TODO: replace with actual address PINATA_JWT: ${{ secrets.PINATA_JWT }} PINATA_DEDICATED_GATEWAY_URL: ${{ secrets.PINATA_DEDICATED_GATEWAY_URL }} PINATA_DEDICATED_GATEWAY_TOKEN: ${{ secrets.PINATA_DEDICATED_GATEWAY_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb39e37ac..fa3812f15 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,8 @@ jobs: TESTNET_EXECUTION_CLIENT_URI: ${{ secrets.TESTNET_EXECUTION_CLIENT_URI }} TESTNET_CONSENSUS_CLIENT_URI: ${{ secrets.TESTNET_CONSENSUS_CLIENT_URI }} TESTNET_KAPI_URI: ${{ secrets.TESTNET_KAPI_URI }} - CSM_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" + CS_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" + CURATED_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F" # TODO: replace with actual address PINATA_JWT: ${{ secrets.PINATA_JWT }} PINATA_DEDICATED_GATEWAY_URL: ${{ secrets.PINATA_DEDICATED_GATEWAY_URL }} PINATA_DEDICATED_GATEWAY_TOKEN: ${{ secrets.PINATA_DEDICATED_GATEWAY_TOKEN }} diff --git a/README.md b/README.md index 04ed74431..0e4315c9c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ cp .env.example .env docker run -ti --env-file .env --rm lidofinance/oracle:{tag} check # 4. Run the Oracle (dry mode by default) -docker run --env-file .env lidofinance/oracle:{tag} accounting # | ejector | csm +docker run --env-file .env lidofinance/oracle:{tag} accounting # | ejector | csm | cm ``` Or checkout [Oracle Operator Manual](https://docs.lido.fi/guides/oracle-operator-manual) for more details. @@ -29,7 +29,8 @@ There are 3 modules in the oracle: - Accounting (accounting) - Valdiators Exit Bus (ejector) -- CSM (csm) +- Community Staking Module (csm) +- Curated Module (cm) ### Accounting module @@ -56,9 +57,9 @@ Work is divided into frames (~8 hours / 75 epochs): - Determines next available validator exit. - Builds validators to exit queue and submits data to Execution Layer. -### CSM module +### Staking Module Oracle -Collects and reports validator attestation rate for node operators. Handles publishing metadata to IPFS for the CSM. +Collects and reports validator attestation rate for node operators. Handles publishing metadata to IPFS for the Staking Module. Work is divided into frames (~28 days / 6300 epochs): - **Data collection**: Processes new epoches and collect attestations. @@ -177,59 +178,59 @@ In manual mode all sleeps are disabled and `ALLOW_REPORTING_IN_BUNKER_MODE` is T ## Env variables -| Name | Description | Required | Example value | -|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------------------| -| `EXECUTION_CLIENT_URI` | URI of the Execution Layer client | True | `http://localhost:8545` | -| `CONSENSUS_CLIENT_URI` | URI of the Consensus Layer client | True | `http://localhost:5052` | -| `KEYS_API_URI` | URI of the Keys API | True | `http://localhost:8080` | -| `LIDO_LOCATOR_ADDRESS` | Address of the Lido contract | True | `0x1...` | -| `CSM_MODULE_ADDRESS` | Address of the CSModule contract | CSM only | `0x1...` | -| `MEMBER_PRIV_KEY` | Private key of the Oracle member account | False | `0x1...` | -| `MEMBER_PRIV_KEY_FILE` | A path to the file contained the private key of the Oracle member account. It takes precedence over `MEMBER_PRIV_KEY` | False | `/app/private_key` | -| `PINATA_JWT` | JWT token to access pinata.cloud IPFS provider | True | `aBcD1234...` | -| `PINATA_JWT_FILE` | A path to a file with a JWT token to access pinata.cloud IPFS provider | True | `/app/pintata_secret` | -| `PINATA_DEDICATED_GATEWAY_URL` | URL of the dedicated Pinata gateway (required for Pinata provider, fallback to public gateway if dedicated fails) | CSM only | `https://gateway.pinata.cloud` | -| `PINATA_DEDICATED_GATEWAY_TOKEN` | Token for accessing dedicated Pinata gateway (required for Pinata provider) | CSM only | `gAT_abc123...` | -| `STORACHA_AUTH_SECRET` | Secret for Storacha IPFS provider | True | `uMGVabc...` | -| `STORACHA_AUTHORIZATION` | Authorization for Storacha IPFS provider | True | `uMGVabc...` | -| `STORACHA_SPACE_DID` | Space DID for Storacha IPFS provider | True | `did:key:z6Mkabc...` | -| `LIDO_IPFS_HOST` | Host to access Lido IPFS cluster | True | `https://ipfs.lido.fi` | -| `LIDO_IPFS_TOKEN` | Bearer token for Lido IPFS cluster authentication | True | `eyJhbG...` | -| `KUBO_HOST` | Host to access running Kubo IPFS node | False | `localhost` | -| `KUBO_RPC_PORT` | Port to access RPC provided by Kubo IPFS node | False | `5001` | -| `KUBO_GATEWAY_PORT` | Port to access gateway provided by Kubo IPFS node | False | `8080` | -| `FINALIZATION_BATCH_MAX_REQUEST_COUNT` | The size of the batch to be finalized per request (The larger the batch size, the more memory of the contract is used but the fewer requests are needed) | False | `1000` | -| `EL_REQUESTS_BATCH_SIZE` | The amount of entities that would be fetched in one request to EL | False | `1000` | -| `ALLOW_REPORTING_IN_BUNKER_MODE` | Allow the Oracle to do report if bunker mode is active | False | `True` | -| `DAEMON` | If False Oracle runs one cycle and ask for manual input to send report. | False | `True` | -| `TX_GAS_ADDITION` | Used to modify gas parameter that used in transaction. (gas = estimated_gas + TX_GAS_ADDITION) | False | `100000` | -| `CYCLE_SLEEP_IN_SECONDS` | The time between cycles of the oracle's activity | False | `12` | -| `MAX_CYCLE_LIFETIME_IN_SECONDS` | The maximum time for a cycle to continue | False | `3000` | -| `SUBMIT_DATA_DELAY_IN_SLOTS` | The difference in slots between submit data transactions from Oracles. It is used to prevent simultaneous sending of transactions and, as a result, transactions revert. | False | `6` | -| `HTTP_REQUEST_TIMEOUT_EXECUTION` | Timeout for HTTP execution layer requests | False | `120` | -| `HTTP_REQUEST_TIMEOUT_CONSENSUS` | Timeout for HTTP consensus layer requests | False | `300` | -| `HTTP_REQUEST_RETRY_COUNT_CONSENSUS` | Total number of retries to fetch data from endpoint for consensus layer requests | False | `5` | -| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS` | The delay http provider sleeps if API is stuck for consensus layer | False | `12` | -| `HTTP_REQUEST_TIMEOUT_KEYS_API` | Timeout for HTTP keys api requests | False | `120` | -| `HTTP_REQUEST_RETRY_COUNT_KEYS_API` | Total number of retries to fetch data from endpoint for keys api requests | False | `300` | -| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_KEYS_API` | The delay http provider sleeps if API is stuck for keys api | False | `300` | -| `HTTP_REQUEST_TIMEOUT_IPFS` | Timeout for HTTP requests to an IPFS provider | False | `30` | -| `HTTP_REQUEST_RETRY_COUNT_IPFS` | Total number of retries to fetch data from an IPFS provider | False | `3` | -| `IPFS_VALIDATE_CID` | Enable/disable CID validation for IPFS operations | False | `True` | -| `EVENTS_SEARCH_STEP` | Maximum length of a range for eth_getLogs method calls | False | `10000` | -| `PRIORITY_FEE_PERCENTILE` | Priority fee percentile from prev block that would be used to send tx | False | `3` | -| `MIN_PRIORITY_FEE` | Min priority fee that would be used to send tx | False | `50000000` | -| `MAX_PRIORITY_FEE` | Max priority fee that would be used to send tx | False | `100000000000` | -| `CSM_ORACLE_MAX_CONCURRENCY` | Max count of dedicated workers for CSM module | False | `2` | -| `CACHE_PATH` | Directory to store cache for CSM module | False | `.` | -| `OPSGENIE_API_KEY` | OpsGenie API key for authentication with the OpsGenie API. Used to send alerts from lido-oracle health-checks. | False | `` | -| `OPSGENIE_API_URL` | Base URL for the OpsGenie API. | False | `http://localhost:8080` | -| `VAULT_PAGINATION_LIMIT` | The limit for getting staking vaults with pagination. Default 1000 | False | `1000` | -| `VAULT_VALIDATOR_STAGES_BATCH_SIZE` | The limit for getting validators stages in one request. Default 100 | False | `100` | +| Name | Description | Required | Example value | +|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|--------------------------------| +| `EXECUTION_CLIENT_URI` | URI of the Execution Layer client | True | `http://localhost:8545` | +| `CONSENSUS_CLIENT_URI` | URI of the Consensus Layer client | True | `http://localhost:5052` | +| `KEYS_API_URI` | URI of the Keys API | True | `http://localhost:8080` | +| `LIDO_LOCATOR_ADDRESS` | Address of the Lido contract | True | `0x1...` | +| `CS_MODULE_ADDRESS` | Address of the Community Staking Module contract | Staking Module only | `0x1...` | +| `MEMBER_PRIV_KEY` | Private key of the Oracle member account | False | `0x1...` | +| `MEMBER_PRIV_KEY_FILE` | A path to the file contained the private key of the Oracle member account. It takes precedence over `MEMBER_PRIV_KEY` | False | `/app/private_key` | +| `PINATA_JWT` | JWT token to access pinata.cloud IPFS provider | True | `aBcD1234...` | +| `PINATA_JWT_FILE` | A path to a file with a JWT token to access pinata.cloud IPFS provider | True | `/app/pintata_secret` | +| `PINATA_DEDICATED_GATEWAY_URL` | URL of the dedicated Pinata gateway (required for Pinata provider, fallback to public gateway if dedicated fails) | Staking Module only | `https://gateway.pinata.cloud` | +| `PINATA_DEDICATED_GATEWAY_TOKEN` | Token for accessing dedicated Pinata gateway (required for Pinata provider) | Staking Module only | `gAT_abc123...` | +| `STORACHA_AUTH_SECRET` | Secret for Storacha IPFS provider | True | `uMGVabc...` | +| `STORACHA_AUTHORIZATION` | Authorization for Storacha IPFS provider | True | `uMGVabc...` | +| `STORACHA_SPACE_DID` | Space DID for Storacha IPFS provider | True | `did:key:z6Mkabc...` | +| `LIDO_IPFS_HOST` | Host to access Lido IPFS cluster | True | `https://ipfs.lido.fi` | +| `LIDO_IPFS_TOKEN` | Bearer token for Lido IPFS cluster authentication | True | `eyJhbG...` | +| `KUBO_HOST` | Host to access running Kubo IPFS node | False | `localhost` | +| `KUBO_RPC_PORT` | Port to access RPC provided by Kubo IPFS node | False | `5001` | +| `KUBO_GATEWAY_PORT` | Port to access gateway provided by Kubo IPFS node | False | `8080` | +| `FINALIZATION_BATCH_MAX_REQUEST_COUNT` | The size of the batch to be finalized per request (The larger the batch size, the more memory of the contract is used but the fewer requests are needed) | False | `1000` | +| `EL_REQUESTS_BATCH_SIZE` | The amount of entities that would be fetched in one request to EL | False | `1000` | +| `ALLOW_REPORTING_IN_BUNKER_MODE` | Allow the Oracle to do report if bunker mode is active | False | `True` | +| `DAEMON` | If False Oracle runs one cycle and ask for manual input to send report. | False | `True` | +| `TX_GAS_ADDITION` | Used to modify gas parameter that used in transaction. (gas = estimated_gas + TX_GAS_ADDITION) | False | `100000` | +| `CYCLE_SLEEP_IN_SECONDS` | The time between cycles of the oracle's activity | False | `12` | +| `MAX_CYCLE_LIFETIME_IN_SECONDS` | The maximum time for a cycle to continue | False | `3000` | +| `SUBMIT_DATA_DELAY_IN_SLOTS` | The difference in slots between submit data transactions from Oracles. It is used to prevent simultaneous sending of transactions and, as a result, transactions revert. | False | `6` | +| `HTTP_REQUEST_TIMEOUT_EXECUTION` | Timeout for HTTP execution layer requests | False | `120` | +| `HTTP_REQUEST_TIMEOUT_CONSENSUS` | Timeout for HTTP consensus layer requests | False | `300` | +| `HTTP_REQUEST_RETRY_COUNT_CONSENSUS` | Total number of retries to fetch data from endpoint for consensus layer requests | False | `5` | +| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS` | The delay http provider sleeps if API is stuck for consensus layer | False | `12` | +| `HTTP_REQUEST_TIMEOUT_KEYS_API` | Timeout for HTTP keys api requests | False | `120` | +| `HTTP_REQUEST_RETRY_COUNT_KEYS_API` | Total number of retries to fetch data from endpoint for keys api requests | False | `300` | +| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_KEYS_API` | The delay http provider sleeps if API is stuck for keys api | False | `300` | +| `HTTP_REQUEST_TIMEOUT_IPFS` | Timeout for HTTP requests to an IPFS provider | False | `30` | +| `HTTP_REQUEST_RETRY_COUNT_IPFS` | Total number of retries to fetch data from an IPFS provider | False | `3` | +| `IPFS_VALIDATE_CID` | Enable/disable CID validation for IPFS operations | False | `True` | +| `EVENTS_SEARCH_STEP` | Maximum length of a range for eth_getLogs method calls | False | `10000` | +| `PRIORITY_FEE_PERCENTILE` | Priority fee percentile from prev block that would be used to send tx | False | `3` | +| `MIN_PRIORITY_FEE` | Min priority fee that would be used to send tx | False | `50000000` | +| `MAX_PRIORITY_FEE` | Max priority fee that would be used to send tx | False | `100000000000` | +| `PERFORMANCE_COLLECTOR_MAX_CONCURRENCY` | Max count of dedicated workers for Performance Collector module | False | `2` | +| `CACHE_PATH` | Directory to store cache for Staking Module Oracle | False | `.` | +| `OPSGENIE_API_KEY` | OpsGenie API key for authentication with the OpsGenie API. Used to send alerts from lido-oracle health-checks. | False | `` | +| `OPSGENIE_API_URL` | Base URL for the OpsGenie API. | False | `http://localhost:8080` | +| `VAULT_PAGINATION_LIMIT` | The limit for getting staking vaults with pagination. Default 1000 | False | `1000` | +| `VAULT_VALIDATOR_STAGES_BATCH_SIZE` | The limit for getting validators stages in one request. Default 100 | False | `100` | ### Mainnet variables > LIDO_LOCATOR_ADDRESS=0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb -> CSM_MODULE_ADDRESS=0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F +> STAKING_MODULE_ADDRESS=0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F > ALLOW_REPORTING_IN_BUNKER_MODE=False ### Alerts @@ -264,25 +265,25 @@ groups: The oracle exposes the following basic metrics: -| Metric name | Description | Labels | -|-----------------------------|-----------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| build_info | Build info | version, branch, commit | -| env_variables_info | Env variables for the app | ACCOUNT, LIDO_LOCATOR_ADDRESS, CSM_MODULE_ADDRESS, FINALIZATION_BATCH_MAX_REQUEST_COUNT, EL_REQUESTS_BATCH_SIZE, MAX_CYCLE_LIFETIME_IN_SECONDS | -| genesis_time | Fetched genesis time from node | | -| account_balance | Fetched account balance from EL | address | -| slot_number | Last fetched slot number from CL | state (`head` or `finalized`) | -| block_number | Last fetched block number from CL | state (`head` or `finalized`) | -| functions_duration | Histogram metric with duration of each main function in the app | name, status | -| cl_requests_duration | Histogram metric with duration of each CL request | endpoint, code, domain | -| keys_api_requests_duration | Histogram metric with duration of each KeysAPI request | endpoint, code, domain | -| keys_api_latest_blocknumber | Latest block number from KeysAPI metadata | | -| transactions_count | Total count of transactions. Success or failure | status | -| member_info | Oracle member info | is_report_member, is_submit_member, is_fast_lane | -| member_last_report_ref_slot | Member last report ref slot | | -| frame_current_ref_slot | Current frame ref slot | | -| frame_deadline_slot | Current frame deadline slot | | -| frame_prev_report_ref_slot | Previous report ref slot | | -| contract_on_pause | Contract on pause | | +| Metric name | Description | Labels | +|-----------------------------|-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| build_info | Build info | version, branch, commit | +| env_variables_info | Env variables for the app | ACCOUNT, LIDO_LOCATOR_ADDRESS, STAKING_MODULE_ADDRESS, FINALIZATION_BATCH_MAX_REQUEST_COUNT, EL_REQUESTS_BATCH_SIZE, MAX_CYCLE_LIFETIME_IN_SECONDS | +| genesis_time | Fetched genesis time from node | | +| account_balance | Fetched account balance from EL | address | +| slot_number | Last fetched slot number from CL | state (`head` or `finalized`) | +| block_number | Last fetched block number from CL | state (`head` or `finalized`) | +| functions_duration | Histogram metric with duration of each main function in the app | name, status | +| cl_requests_duration | Histogram metric with duration of each CL request | endpoint, code, domain | +| keys_api_requests_duration | Histogram metric with duration of each KeysAPI request | endpoint, code, domain | +| keys_api_latest_blocknumber | Latest block number from KeysAPI metadata | | +| transactions_count | Total count of transactions. Success or failure | status | +| member_info | Oracle member info | is_report_member, is_submit_member, is_fast_lane | +| member_last_report_ref_slot | Member last report ref slot | | +| frame_current_ref_slot | Current frame ref slot | | +| frame_deadline_slot | Current frame deadline slot | | +| frame_prev_report_ref_slot | Previous report ref slot | | +| contract_on_pause | Contract on pause | | Interaction with external providers: @@ -313,14 +314,12 @@ Special metrics for ejector oracle: | ejector_max_withdrawal_epoch | Max withdrawal epoch among all Lido validators on CL | | | ejector_validators_count_to_eject | Validators count to eject | | -Special metrics for CSM oracle: +Special metrics for Staking Module oracle: -| Metric name | Description | Labels | -|---------------------------------|----------------------------------------|--------| -| csm_current_frame_range_l_epoch | Left epoch of the current frame range | | -| csm_current_frame_range_r_epoch | Right epoch of the current frame range | | -| csm_unprocessed_epochs_count | Unprocessed epochs count | | -| csm_min_unprocessed_epoch | Minimum unprocessed epoch | | +| Metric name | Description | Labels | +|--------------------------------------------|----------------------------------------|--------| +| staking_module_current_frame_range_l_epoch | Left epoch of the current frame range | | +| staking_module_current_frame_range_r_epoch | Right epoch of the current frame range | | # Development diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..dc9fd3728 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,149 @@ +# Usage: +# Local development: docker compose up -d +# External PostgreSQL: PERFORMANCE_DB_HOST=your-host docker compose up -d performance-collector performance-web +# Run init-db only: docker compose up init-db + +x-healthcheck-defaults: &healthcheck-defaults + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + +x-performance-db-env: &performance-db-env + PERFORMANCE_DB_HOST: ${PERFORMANCE_DB_HOST:-postgres} + PERFORMANCE_DB_PORT: ${PERFORMANCE_DB_PORT:-5432} + PERFORMANCE_DB_NAME: ${PERFORMANCE_DB_NAME:-performance} + PERFORMANCE_DB_USER: ${PERFORMANCE_DB_USER:-performance} + PERFORMANCE_DB_PASSWORD: ${PERFORMANCE_DB_PASSWORD:-performance} + +services: + # --------------------------------------------------------------------------- + # Database Layer + # --------------------------------------------------------------------------- + + postgres: + image: postgres:15 + container_name: staking-module-oracle-postgres + environment: + POSTGRES_USER: ${PERFORMANCE_DB_ADMIN_USER:-postgres} + POSTGRES_PASSWORD: ${PERFORMANCE_DB_ADMIN_PASSWORD:-postgres} + POSTGRES_DB: postgres + ports: + - "${POSTGRES_PORT:-5432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PERFORMANCE_DB_ADMIN_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # One-time database initialization (creates user and database) + # For external PostgreSQL: set PERFORMANCE_DB_HOST to your external host + init-db: + image: postgres:15 + container_name: staking-module-oracle-init-db + depends_on: + postgres: + condition: service_healthy + environment: + <<: *performance-db-env + PERFORMANCE_DB_ADMIN_USER: ${PERFORMANCE_DB_ADMIN_USER:-postgres} + PERFORMANCE_DB_ADMIN_PASSWORD: ${PERFORMANCE_DB_ADMIN_PASSWORD:-postgres} + volumes: + - ./scripts:/scripts:ro + entrypoint: ["/bin/bash", "/scripts/init_performance_db.sh"] + restart: "no" + + # --------------------------------------------------------------------------- + # Performance Sidecar + # --------------------------------------------------------------------------- + + performance-collector: + build: . + container_name: staking-module-oracle-performance-collector + command: ["-m", "src.main", "performance_collector"] + depends_on: + init-db: + condition: service_completed_successfully + environment: + <<: *performance-db-env + CONSENSUS_CLIENT_URI: ${CONSENSUS_CLIENT_URI} + MAX_CYCLE_LIFETIME_IN_SECONDS: ${MAX_CYCLE_LIFETIME_IN_SECONDS:-60000} + ports: + - "${PERFORMANCE_COLLECTOR_PROMETHEUS_PORT:-9000}:9000" + - "${PERFORMANCE_COLLECTOR_HEALTH_PORT:-9010}:9010" + healthcheck: + <<: *healthcheck-defaults + test: ["CMD", "curl", "-f", "http://localhost:9010/healthcheck"] + + performance-web: + build: . + container_name: staking-module-oracle-performance-web + command: ["-m", "src.main", "performance_web_server"] + depends_on: + init-db: + condition: service_completed_successfully + environment: + <<: *performance-db-env + ports: + - "${PERFORMANCE_WEB_PORT:-9020}:9020" + healthcheck: + <<: *healthcheck-defaults + test: ["CMD", "curl", "-f", "http://localhost:9020/health"] + + # --------------------------------------------------------------------------- + # Staking Module Oracles + # --------------------------------------------------------------------------- + + csm-oracle: + build: . + container_name: csm-oracle + command: ["-m", "src.main", "csm"] + environment: + CS_MODULE_ADDRESS: ${CS_MODULE_ADDRESS} + CONSENSUS_CLIENT_URI: ${CONSENSUS_CLIENT_URI} + EXECUTION_CLIENT_URI: ${EXECUTION_CLIENT_URI} + KEYS_API_URI: ${KEYS_API_URI} + PERFORMANCE_COLLECTOR_URI: http://performance-web:9020/ + LIDO_IPFS_HOST: ${LIDO_IPFS_HOST:-} + LIDO_IPFS_TOKEN: ${LIDO_IPFS_TOKEN:-} + PROMETHEUS_PORT: 9000 + HEALTHCHECK_SERVER_PORT: 9010 + ports: + - "${CSM_ORACLE_PROMETHEUS_PORT:-9001}:9000" + - "${CSM_ORACLE_HEALTH_PORT:-9011}:9010" + restart: unless-stopped + healthcheck: + <<: *healthcheck-defaults + test: ["CMD", "curl", "-f", "http://localhost:9010/healthcheck"] + +# cm-oracle: +# build: . +# container_name: cm-oracle +# command: ["-m", "src.main", "cm"] +# environment: +# CURATED_MODULE_ADDRESS: ${CURATED_MODULE_ADDRESS} +# CONSENSUS_CLIENT_URI: ${CONSENSUS_CLIENT_URI} +# EXECUTION_CLIENT_URI: ${EXECUTION_CLIENT_URI} +# KEYS_API_URI: ${KEYS_API_URI} +# PERFORMANCE_COLLECTOR_URI: http://performance-web:9020/ +# LIDO_IPFS_HOST: ${LIDO_IPFS_HOST:-} +# LIDO_IPFS_TOKEN: ${LIDO_IPFS_TOKEN:-} +# PROMETHEUS_PORT: 9000 +# HEALTHCHECK_SERVER_PORT: 9010 +# ports: +# - "${CSM_ORACLE_PROMETHEUS_PORT:-9002}:9000" +# - "${CSM_ORACLE_HEALTH_PORT:-9012}:9010" +# restart: unless-stopped +# healthcheck: +# <<: *healthcheck-defaults +# test: ["CMD", "curl", "-f", "http://localhost:9010/healthcheck"] + +volumes: + postgres-data: + +networks: + default: + name: oracle-network diff --git a/docs/development.md b/docs/development.md index 3d816563e..d6dbb6006 100644 --- a/docs/development.md +++ b/docs/development.md @@ -40,14 +40,15 @@ export KEYS_API_URI=... export LIDO_LOCATOR_ADDRESS=... ``` -Required variables for CSM module +Required variables for Staking Module Oracle ```bash export EXECUTION_CLIENT_URI=... export CONSENSUS_CLIENT_URI=... export KEYS_API_URI=... export LIDO_LOCATOR_ADDRESS=... -export CSM_MODULE_ADDRESS=... +export CS_MODULE_ADDRESS=... +export CURATED_MODULE_ADDRESS=... export MAX_CYCLE_LIFETIME_IN_SECONDS=60000 # Reasonable high value to make sure the oracle has enough time to process the whole frame. ``` @@ -67,6 +68,7 @@ Where `` is one of: - `accounting` - `ejector` - `csm` +- `cm` - `check` ## Code quality diff --git a/scripts/init_performance_db.sh b/scripts/init_performance_db.sh new file mode 100755 index 000000000..8bb0c5e25 --- /dev/null +++ b/scripts/init_performance_db.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Script to initialize the Performance database. +# This creates the required user and database for the Performance Collector and Web Server. +# + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Admin connection settings +DB_HOST="${PERFORMANCE_DB_HOST:-localhost}" +DB_PORT="${PERFORMANCE_DB_PORT:-5432}" +ADMIN_USER="${PERFORMANCE_DB_ADMIN_USER:-postgres}" +ADMIN_PASSWORD="${PERFORMANCE_DB_ADMIN_PASSWORD}" + +# New user/database settings +DB_NAME="${PERFORMANCE_DB_NAME:-performance}" +DB_USER="${PERFORMANCE_DB_USER:-performance}" +DB_PASSWORD="${PERFORMANCE_DB_PASSWORD:-performance}" + +# Validate required variables +if [ -z "$ADMIN_PASSWORD" ]; then + log_error "PERFORMANCE_DB_ADMIN_PASSWORD is required" + exit 1 +fi + +log_info "Connecting to PostgreSQL at ${DB_HOST}:${DB_PORT} as ${ADMIN_USER}" + +# Export password for psql +export PGPASSWORD="$ADMIN_PASSWORD" + +# Function to run SQL command +run_sql() { + psql -h "$DB_HOST" -p "$DB_PORT" -U "$ADMIN_USER" -d postgres -tAc "$1" +} + +# Check if user exists +user_exists() { + local result + result=$(run_sql "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" 2>/dev/null) + [ "$result" = "1" ] +} + +# Check if database exists +db_exists() { + local result + result=$(run_sql "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" 2>/dev/null) + [ "$result" = "1" ] +} + +# Create user if not exists +if user_exists; then + log_warn "User '$DB_USER' already exists, skipping creation" +else + log_info "Creating user '$DB_USER'..." + run_sql "CREATE USER $DB_USER WITH PASSWORD '$DB_PASSWORD'" + log_info "User '$DB_USER' created successfully" +fi + +# Create database if not exists +if db_exists; then + log_warn "Database '$DB_NAME' already exists, skipping creation" +else + log_info "Creating database '$DB_NAME' with owner '$DB_USER'..." + run_sql "CREATE DATABASE $DB_NAME OWNER $DB_USER" + log_info "Database '$DB_NAME' created successfully" +fi + +# Grant privileges +log_info "Granting privileges..." +run_sql "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER" + +# Connect to the new database and grant schema privileges +export PGPASSWORD="$ADMIN_PASSWORD" +psql -h "$DB_HOST" -p "$DB_PORT" -U "$ADMIN_USER" -d "$DB_NAME" -c "GRANT ALL ON SCHEMA public TO $DB_USER" + +log_info "Database initialization completed successfully!" +log_info "" +log_info "Connection details:" +log_info " Host: $DB_HOST" +log_info " Port: $DB_PORT" +log_info " Database: $DB_NAME" +log_info " User: $DB_USER" diff --git a/src/constants.py b/src/constants.py index 5d5e730fd..53d000e9a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,7 +1,6 @@ from packaging.version import Version from src.types import Gwei - # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc FAR_FUTURE_EPOCH = 2**64 - 1 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters-1 @@ -50,7 +49,7 @@ SHARE_RATE_PRECISION_E27 = 10**PRECISION_E27 TOTAL_BASIS_POINTS = 10000 -# Lido CSM constants for network performance calculation +# Lido Staking Module constants for network performance calculation ATTESTATIONS_WEIGHT = 54 BLOCKS_WEIGHT = 8 SYNC_WEIGHT = 2 @@ -62,7 +61,7 @@ UINT256_MAX = 2**256 - 1 ALLOWED_KAPI_VERSION = Version('1.5.0') -CSM_STATE_VERSION = 1 -CSM_LOGS_VERSION = 1 +STAKING_MODULE_STATE_VERSION = 1 +STAKING_MODULE_LOGS_VERSION = 1 GENESIS_VALIDATORS_ROOT = bytes([0] * 32) # all zeros for deposits diff --git a/src/main.py b/src/main.py index 1270c40a8..7a39d25b6 100644 --- a/src/main.py +++ b/src/main.py @@ -1,215 +1,64 @@ import sys -from typing import Iterator, cast - -from packaging.version import Version -from prometheus_client import start_http_server -from web3_multi_provider.metrics import init_metrics +from decimal import getcontext -from src import constants, variables +from src import variables from src.constants import PRECISION_E27 -from src.metrics.healthcheck_server import start_pulse_server from src.metrics.logging import logging -from src.metrics.prometheus.basic import BUILD_INFO, ENV_VARIABLES_INFO -from src.modules.accounting.accounting import Accounting -from src.modules.checks.checks_module import ChecksModule -from src.modules.csm.csm import CSOracle -from src.modules.ejector.ejector import Ejector -from src.providers.ipfs import IPFSProvider, Kubo, LidoIPFS, Pinata, Storacha -from src.modules.performance.collector.collector import PerformanceCollector from src.types import OracleModule -from src.utils.build import get_build_info -from src.utils.exception import IncompatibleException -from src.web3py.contract_tweak import tweak_w3_contracts -from src.web3py.extensions import ( - ConsensusClientModule, - FallbackProviderModule, - IPFS, - KeysAPIClientModule, - LazyCSM, - LidoContracts, - LidoValidatorsProvider, - TransactionUtils, -) -from src.web3py.extensions.performance import PerformanceClientModule -from src.web3py.types import Web3 -from decimal import getcontext - -getcontext().prec = PRECISION_E27 logger = logging.getLogger(__name__) +getcontext().prec = PRECISION_E27 -def main(module_name: OracleModule): - build_info = get_build_info() - logger.info({ - 'msg': 'Oracle startup.', - 'variables': { - **build_info, - 'module': module_name, - **variables.PUBLIC_ENV_VARS, - }, - }) - ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS) - BUILD_INFO.info(build_info) - - logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) - start_pulse_server() - - logger.info({'msg': f'Start http server with prometheus metrics on port {variables.PROMETHEUS_PORT}'}) - start_http_server(variables.PROMETHEUS_PORT) - - logger.info({'msg': 'Initialize multi web3 provider.'}) - web3 = Web3(FallbackProviderModule( - variables.EXECUTION_CLIENT_URI, - request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION}, - cache_allowed_requests=True, - )) - - logger.info({'msg': 'Modify web3 with custom contract function call.'}) - tweak_w3_contracts(web3) - - logger.info({'msg': 'Initialize consensus client.'}) - cc = ConsensusClientModule(variables.CONSENSUS_CLIENT_URI, web3) - - logger.info({'msg': 'Initialize keys api client.'}) - kac = KeysAPIClientModule(variables.KEYS_API_URI, web3) - - logger.info({'msg': 'Initialize IPFS providers.'}) - ipfs = IPFS(web3, ipfs_providers(), retries=variables.HTTP_REQUEST_RETRY_COUNT_IPFS) - - logger.info({'msg': 'Initialize Performance Collector client.'}) - performance = PerformanceClientModule(variables.PERFORMANCE_COLLECTOR_URI) - - logger.info({'msg': 'Check configured providers.'}) - if Version(kac.get_status().appVersion) < constants.ALLOWED_KAPI_VERSION: - raise IncompatibleException(f'Incompatible KAPI version. Required >= {constants.ALLOWED_KAPI_VERSION}.') - - check_providers_chain_ids(web3, cc, kac) - - web3.attach_modules({ - 'lido_contracts': LidoContracts, - 'lido_validators': LidoValidatorsProvider, - 'transaction': TransactionUtils, - 'csm': LazyCSM, - 'cc': lambda: cc, # type: ignore[dict-item] - 'kac': lambda: kac, # type: ignore[dict-item] - 'ipfs': lambda: ipfs, # type: ignore[dict-item] - 'performance': lambda: performance, # type: ignore[dict-item] - }) - - logger.info({'msg': 'Initialize prometheus metrics.'}) - init_metrics() - - instance: Accounting | Ejector | CSOracle | PerformanceCollector - if module_name == OracleModule.ACCOUNTING: - logger.info({'msg': 'Initialize Accounting module.'}) - instance = Accounting(web3) - elif module_name == OracleModule.EJECTOR: - logger.info({'msg': 'Initialize Ejector module.'}) - instance = Ejector(web3) - elif module_name == OracleModule.CSM: - logger.info({'msg': 'Initialize CSM performance oracle module.'}) - instance = CSOracle(web3) - elif module_name == OracleModule.PERFORMANCE_COLLECTOR: - logger.info({'msg': 'Initialize Performance Collector module.'}) - # FIXME: web3 object is overkill. only CONSENSUS_CLIENT_URI needed here. - instance = PerformanceCollector(web3) - else: - raise ValueError(f'Unexpected arg: {module_name=}.') - - if module_name != OracleModule.PERFORMANCE_COLLECTOR: - if hasattr(instance, 'check_contract_configs'): - instance.check_contract_configs() - - if variables.DAEMON: - instance.run_as_daemon() - else: - instance.cycle_handler() - - -def check(): - logger.info({'msg': 'Check oracle is ready to work in the current environment.'}) - - return ChecksModule().execute_module() - - -def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): - keys_api_chain_id = kac.check_providers_consistency() - consensus_chain_id = cc.check_providers_consistency() - execution_chain_id = cast(FallbackProviderModule, web3.provider).check_providers_consistency() - - if execution_chain_id == consensus_chain_id == keys_api_chain_id: - return - - raise IncompatibleException( - 'Different chain ids detected:\n' - f'Execution chain id: {execution_chain_id}\n' - f'Consensus chain id: {consensus_chain_id}\n' - f'Keys API chain id: {keys_api_chain_id}\n' - ) - - -def ipfs_providers() -> Iterator[IPFSProvider]: - if variables.KUBO_HOST: - yield Kubo( - variables.KUBO_HOST, - variables.KUBO_RPC_PORT, - variables.KUBO_GATEWAY_PORT, - timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, - ) - - if variables.PINATA_JWT and variables.PINATA_DEDICATED_GATEWAY_URL and variables.PINATA_DEDICATED_GATEWAY_TOKEN: - yield Pinata( - variables.PINATA_JWT, - timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, - dedicated_gateway_url=variables.PINATA_DEDICATED_GATEWAY_URL, - dedicated_gateway_token=variables.PINATA_DEDICATED_GATEWAY_TOKEN, - ) - - if ( - variables.STORACHA_AUTH_SECRET and - variables.STORACHA_AUTHORIZATION and - variables.STORACHA_SPACE_DID - ): - yield Storacha( - variables.STORACHA_AUTH_SECRET, - variables.STORACHA_AUTHORIZATION, - variables.STORACHA_SPACE_DID, - timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, - ) - - if variables.LIDO_IPFS_HOST and variables.LIDO_IPFS_TOKEN: - yield LidoIPFS( - variables.LIDO_IPFS_HOST, - variables.LIDO_IPFS_TOKEN, - timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, - ) - -if __name__ == '__main__': - module_name_arg = sys.argv[-1] - if module_name_arg not in OracleModule: - msg = f'Last arg should be one of {[str(item) for item in OracleModule]}, received {module_name_arg}.' - logger.error({'msg': msg}) - raise ValueError(msg) - - module = OracleModule(module_name_arg) +def main(module: OracleModule): + # pylint: disable=import-outside-toplevel,too-many-return-statements if module is OracleModule.CHECK: errors = variables.check_uri_required_variables() variables.raise_from_errors(errors) - sys.exit(check()) + from src.modules.checks import entrypoint as check_entrypoint + return sys.exit(check_entrypoint.run()) if module is OracleModule.PERFORMANCE_WEB_SERVER: - from src.modules.performance.web.server import serve errors = variables.check_perf_web_server_required_variables() variables.raise_from_errors(errors) - logger.info({'msg': f'Starting Performance Web Server on port {variables.PERFORMANCE_WEB_SERVER_API_PORT}'}) - sys.exit(serve()) + from src.modules.sidecars.performance.web import entrypoint as web_entrypoint + return sys.exit(web_entrypoint.run()) if module is OracleModule.PERFORMANCE_COLLECTOR: errors = variables.check_perf_collector_required_variables() - else: - errors = variables.check_all_required_variables(module) + variables.raise_from_errors(errors) + from src.modules.sidecars.performance.collector import entrypoint as collector_entrypoint + return collector_entrypoint.run() + + errors = variables.check_all_required_variables(module) variables.raise_from_errors(errors) - main(module) + + if module is OracleModule.ACCOUNTING: + from src.modules.oracles.accounting import entrypoint as accounting_entrypoint + return accounting_entrypoint.run() + + if module is OracleModule.EJECTOR: + from src.modules.oracles.ejector import entrypoint as ejector_entrypoint + return ejector_entrypoint.run() + + if module is OracleModule.CSM: + from src.modules.oracles.staking_modules.community_staking import entrypoint as csm_entrypoint + return csm_entrypoint.run() + + if module is OracleModule.CM: + from src.modules.oracles.staking_modules.curated import entrypoint as cm_entrypoint + return cm_entrypoint.run() + + raise ValueError(f'Unknown module {module}') + + +if __name__ == '__main__': + module_name_arg = sys.argv[-1] + if module_name_arg not in OracleModule: + msg = f'Last arg should be one of {[str(item) for item in OracleModule]}, received {module_name_arg}.' + logger.error({'msg': msg}) + raise ValueError(msg) + + main(OracleModule(module_name_arg)) diff --git a/src/metrics/prometheus/csm.py b/src/metrics/prometheus/csm.py deleted file mode 100644 index 87bbaca83..000000000 --- a/src/metrics/prometheus/csm.py +++ /dev/null @@ -1,29 +0,0 @@ -from prometheus_client import Gauge - -from src.variables import PROMETHEUS_PREFIX - - -CSM_CURRENT_FRAME_RANGE_L_EPOCH = Gauge( - "csm_current_frame_range_l_epoch", - "Left epoch of the current frame range", - namespace=PROMETHEUS_PREFIX, -) - -CSM_CURRENT_FRAME_RANGE_R_EPOCH = Gauge( - "csm_current_frame_range_r_epoch", - "Right epoch of the current frame range", - namespace=PROMETHEUS_PREFIX, -) - -CSM_UNPROCESSED_EPOCHS_COUNT = Gauge( - "csm_unprocessed_epochs_count", - "Unprocessed epochs count", - namespace=PROMETHEUS_PREFIX, -) - - -CSM_MIN_UNPROCESSED_EPOCH = Gauge( - "csm_min_unprocessed_epoch", - "Minimum unprocessed epoch", - namespace=PROMETHEUS_PREFIX, -) diff --git a/src/metrics/prometheus/staking_module.py b/src/metrics/prometheus/staking_module.py new file mode 100644 index 000000000..b570985f3 --- /dev/null +++ b/src/metrics/prometheus/staking_module.py @@ -0,0 +1,16 @@ +from prometheus_client import Gauge + +from src.variables import PROMETHEUS_PREFIX + + +STAKING_MODULE_CURRENT_FRAME_RANGE_L_EPOCH = Gauge( + "staking_module_current_frame_range_l_epoch", + "Left epoch of the current frame range", + namespace=PROMETHEUS_PREFIX, +) + +STAKING_MODULE_CURRENT_FRAME_RANGE_R_EPOCH = Gauge( + "staking_module_current_frame_range_r_epoch", + "Right epoch of the current frame range", + namespace=PROMETHEUS_PREFIX, +) diff --git a/src/modules/checks/checks_module.py b/src/modules/checks/checks_module.py index 59dd9ee09..4eeed034b 100644 --- a/src/modules/checks/checks_module.py +++ b/src/modules/checks/checks_module.py @@ -11,8 +11,8 @@ class ChecksModule: - Keys API service if LIDO_LOCATOR address provided - Checks configs in Accounting module and Ejector module - if CSM_MODULE_ADDRESS provided - - Checks configs in CSM oracle module + if CS_MODULE_ADDRESS and CURATED_MODULE_ADDRESS provided + - Checks configs in Staking Module oracle module - Checks with special blockstamp value (6300 slots in the past) """ def execute_module(self): diff --git a/src/modules/checks/entrypoint.py b/src/modules/checks/entrypoint.py new file mode 100644 index 000000000..eaad577de --- /dev/null +++ b/src/modules/checks/entrypoint.py @@ -0,0 +1,10 @@ +import logging + +from src.modules.checks.checks_module import ChecksModule + +logger = logging.getLogger(__name__) + + +def run() -> int: + logger.info({'msg': 'Check oracle is ready to work in the current environment.'}) + return ChecksModule().execute_module() diff --git a/src/modules/checks/suites/common.py b/src/modules/checks/suites/common.py index 46417b5a6..7518ad009 100644 --- a/src/modules/checks/suites/common.py +++ b/src/modules/checks/suites/common.py @@ -1,10 +1,12 @@ """Common checks""" import pytest +from src import variables -from src.main import check_providers_chain_ids as chain_ids_check # rename to not conflict with test -from src.modules.accounting.accounting import Accounting -from src.modules.ejector.ejector import Ejector -from src.modules.csm.csm import CSOracle +from src.modules.oracles.common.runtime import check_providers_chain_ids as chain_ids_check # rename to not conflict with test +from src.modules.oracles.accounting.accounting import Accounting +from src.modules.oracles.ejector.ejector import Ejector +from src.modules.oracles.staking_modules.community_staking.csm import CSPerformanceOracle +from src.modules.oracles.staking_modules.curated.cm import CMPerformanceOracle @pytest.fixture() @@ -14,9 +16,15 @@ def skip_locator(web3): @pytest.fixture() -def skip_csm(web3): - if not hasattr(web3, 'csm'): - pytest.skip('CSM_MODULE_ADDRESS is not set') +def skip_csm(): + if not variables.CS_MODULE_ADDRESS: + pytest.skip('CS_MODULE_ADDRESS is not set') + + +@pytest.fixture() +def skip_cm(): + if not variables.CURATED_MODULE_ADDRESS: + pytest.skip('CURATED_MODULE_ADDRESS is not set') @pytest.fixture() @@ -30,8 +38,13 @@ def ejector(web3, skip_locator): @pytest.fixture() -def csm(web3, skip_locator, skip_csm): - return CSOracle(web3) +def csm(web3_cs_module, skip_locator, skip_csm): + return CSPerformanceOracle(web3_cs_module) + + +@pytest.fixture() +def cm(web3_curated_module, skip_locator, skip_cm): + return CMPerformanceOracle(web3_curated_module) def check_providers_chain_ids(web3): @@ -52,3 +65,8 @@ def check_ejector_contract_configs(ejector): def check_csm_contract_configs(csm): """Make sure csm contract configs are valid""" csm.check_contract_configs() + + +def check_cm_contract_configs(cm): + """Make sure cm contract configs are valid""" + cm.check_contract_configs() diff --git a/src/modules/checks/suites/conftest.py b/src/modules/checks/suites/conftest.py index 06cf97977..39b9be6a3 100644 --- a/src/modules/checks/suites/conftest.py +++ b/src/modules/checks/suites/conftest.py @@ -1,4 +1,5 @@ import pytest +from unittest.mock import patch from _pytest._io import TerminalWriter from xdist import is_xdist_controller # type: ignore[import] from xdist.dsession import TerminalDistReporter # type: ignore[import] @@ -8,6 +9,7 @@ from src.utils.blockstamp import build_blockstamp from src.utils.api import opsgenie_api from src.utils.slot import get_reference_blockstamp +from src.modules.oracles.common.runtime import OracleWeb3Config, build_oracle_web3 from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import ( ConsensusClientModule, @@ -16,7 +18,6 @@ TransactionUtils, LidoContracts, FallbackProviderModule, - CSM, ) from src.web3py.types import Web3 @@ -28,6 +29,7 @@ @pytest.fixture() def web3(): + """Basic web3 fixture without staking module contracts.""" web3 = Web3( FallbackProviderModule( variables.EXECUTION_CLIENT_URI, request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION} @@ -47,12 +49,34 @@ def web3(): ) if variables.LIDO_LOCATOR_ADDRESS: web3.attach_modules({'lido_contracts': LidoContracts}) - if variables.CSM_MODULE_ADDRESS: - web3.attach_modules({'csm': CSM}) return web3 +@pytest.fixture() +def web3_cs_module(): + """Web3 fixture configured for CS module (reuses CSM entrypoint logic).""" + with patch.object(variables, 'CURATED_MODULE_ADDRESS', None): + return build_oracle_web3(OracleWeb3Config( + use_lido_contracts=False, + use_staking_module_contracts=True, + use_ipfs=False, + use_performance_client=False, + )) + + +@pytest.fixture() +def web3_curated_module(): + """Web3 fixture configured for Curated module (reuses CM entrypoint logic).""" + with patch.object(variables, 'CS_MODULE_ADDRESS', None): + return build_oracle_web3(OracleWeb3Config( + use_lido_contracts=False, + use_staking_module_contracts=True, + use_ipfs=False, + use_performance_client=False, + )) + + @pytest.fixture( params=[ pytest.param(0, id="Finalized blockstamp"), @@ -60,7 +84,7 @@ def web3(): pytest.param( 6300, id="Blockstamp CSM frame ago", - marks=pytest.mark.skipif(variables.CSM_MODULE_ADDRESS is None, reason="CSM_MODULE_ADDRESS is not set"), + marks=pytest.mark.skipif(variables.CS_MODULE_ADDRESS is None and variables.CURATED_MODULE_ADDRESS is None, reason="Neither CS_MODULE_ADDRESS nor CURATED_MODULE_ADDRESS is set"), ), ] ) diff --git a/src/modules/checks/suites/ipfs.py b/src/modules/checks/suites/ipfs.py index 2bcc89e7c..272f7a967 100644 --- a/src/modules/checks/suites/ipfs.py +++ b/src/modules/checks/suites/ipfs.py @@ -5,8 +5,8 @@ import time from typing import Callable, Any -from src.main import ipfs_providers -from src.providers.ipfs import Pinata, Storacha, LidoIPFS +from src.modules.oracles.common.runtime import ipfs_providers +from src.providers.ipfs import LidoIPFS, Pinata, Storacha REQUIRED_PROVIDERS = (Pinata, Storacha, LidoIPFS) diff --git a/src/modules/common/daemon_module.py b/src/modules/common/daemon_module.py new file mode 100644 index 000000000..866282797 --- /dev/null +++ b/src/modules/common/daemon_module.py @@ -0,0 +1,104 @@ +import logging +import signal +import time +from abc import ABC, abstractmethod +from dataclasses import asdict +from typing import ContextManager + +from timeout_decorator import timeout + +from src import variables +from src.metrics.healthcheck_server import pulse +from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER +from src.modules.common.types import ModuleExecuteDelay +from src.types import BlockStamp, BlockRoot, SlotNumber +from src.utils.blockstamp import build_blockstamp + +logger = logging.getLogger(__name__) + + +class DaemonModule(ABC): + """ + Base class for daemon-like modules. + + Provides common functionality for: + - Running in daemon mode + - Cycle handling + - Getting last finalized slot + - Timeout management + """ + + _slot_threshold: SlotNumber = SlotNumber(0) + + def run_as_daemon(self): + """Starts module in daemon mode with infinite loop""" + logger.info({'msg': 'Run module as daemon.'}) + while True: + logger.debug({'msg': 'Startup new cycle.'}) + self.cycle_handler() + + def cycle_handler(self): + """Handles one daemon module cycle""" + self._cycle() + self._sleep_cycle() + + @timeout(variables.MAX_CYCLE_LIFETIME_IN_SECONDS) + def _cycle(self): + """ + Main cycle logic: gets last finalized slot and executes module business logic + """ + with self.exception_handler(): + blockstamp = self._receive_last_finalized_slot() + + if blockstamp.slot_number <= self._slot_threshold: + logger.info({ + 'msg': 'Skipping the report. Waiting for new finalized slot.', + 'slot_threshold': self._slot_threshold, + }) + return + + self.run_cycle(blockstamp) + + @staticmethod + def _reset_cycle_timeout(): + """Resets timeout timer for current cycle""" + logger.info({'msg': f'Reset running cycle timeout to {variables.MAX_CYCLE_LIFETIME_IN_SECONDS} seconds'}) + signal.setitimer(signal.ITIMER_REAL, variables.MAX_CYCLE_LIFETIME_IN_SECONDS) + pulse() + + @staticmethod + def _sleep_cycle(): + """Handles sleeping between cycles based on configured cycle sleep time""" + logger.info({'msg': f'Cycle end. Sleeping for {variables.CYCLE_SLEEP_IN_SECONDS} seconds.'}) + time.sleep(variables.CYCLE_SLEEP_IN_SECONDS) + + def _receive_last_finalized_slot(self) -> BlockStamp: + """Gets last finalized BlockStamp""" + cc = self._get_consensus_client() + block_root = BlockRoot(cc.get_block_root('finalized').root) + block_details = cc.get_block_details(block_root) + bs = build_blockstamp(block_details) + logger.info({'msg': 'Fetch last finalized BlockStamp.', 'value': asdict(bs)}) + ORACLE_SLOT_NUMBER.labels('finalized').set(bs.slot_number) + ORACLE_BLOCK_NUMBER.labels('finalized').set(bs.block_number) + return bs + + def run_cycle(self, last_finalized_blockstamp: BlockStamp): + """Base logic for daemon module cycle execution""" + logger.info({'msg': 'Execute module.', 'value': last_finalized_blockstamp}) + result = self.execute_module(last_finalized_blockstamp) + pulse() + if result is ModuleExecuteDelay.NEXT_FINALIZED_EPOCH: + self._slot_threshold = last_finalized_blockstamp.slot_number + + @abstractmethod + def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecuteDelay: + """Executes module business logic for given blockstamp""" + + @abstractmethod + def _get_consensus_client(self): + """Returns consensus client for blockchain data access""" + + @abstractmethod + def exception_handler(self) -> ContextManager[None]: + """Context manager for cycle exception handling""" diff --git a/src/modules/submodules/types.py b/src/modules/common/types.py similarity index 77% rename from src/modules/submodules/types.py rename to src/modules/common/types.py index afcf860e2..21f1cc6a2 100644 --- a/src/modules/submodules/types.py +++ b/src/modules/common/types.py @@ -1,8 +1,24 @@ from dataclasses import dataclass +from enum import Enum +from typing import NewType from src.types import SlotNumber -ZERO_HASH = bytes([0]*32) +# Constants +ZERO_HASH = bytes([0] * 32) + + +class ModuleExecuteDelay(Enum): + """Signals from execute_module method""" + NEXT_SLOT = 0 + NEXT_FINALIZED_EPOCH = 1 + + +@dataclass(frozen=True) +class ChainConfig: + slots_per_epoch: int + seconds_per_slot: int + genesis_time: int @dataclass @@ -18,16 +34,11 @@ class MemberInfo: current_frame_consensus_report: bytes -@dataclass(frozen=True) -class ChainConfig: - slots_per_epoch: int - seconds_per_slot: int - genesis_time: int - @dataclass(frozen=True) class ConsensusGenesisConfig: genesis_time: int + @dataclass(frozen=True) class CurrentFrame: # Order is important! @@ -40,4 +51,4 @@ class FrameConfig: # Order is important! initial_epoch: int epochs_per_frame: int - fast_lane_length_slots: int + fast_lane_length_slots: int \ No newline at end of file diff --git a/src/modules/accounting/__init__.py b/src/modules/oracles/__init__.py similarity index 100% rename from src/modules/accounting/__init__.py rename to src/modules/oracles/__init__.py diff --git a/src/modules/accounting/third_phase/__init__.py b/src/modules/oracles/accounting/__init__.py similarity index 100% rename from src/modules/accounting/third_phase/__init__.py rename to src/modules/oracles/accounting/__init__.py diff --git a/src/modules/accounting/accounting.py b/src/modules/oracles/accounting/accounting.py similarity index 97% rename from src/modules/accounting/accounting.py rename to src/modules/oracles/accounting/accounting.py index 62efc4d49..365c90382 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/oracles/accounting/accounting.py @@ -16,9 +16,9 @@ VAULTS_TOTAL_VALUE, ) from src.metrics.prometheus.duration_meter import duration_meter -from src.modules.accounting.third_phase.extra_data import ExtraDataService -from src.modules.accounting.third_phase.types import ExtraData, FormatList -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.third_phase.extra_data import ExtraDataService +from src.modules.oracles.accounting.third_phase.types import ExtraData, FormatList +from src.modules.oracles.accounting.types import ( AccountingProcessingState, BunkerMode, FinalizationShareRate, @@ -32,12 +32,11 @@ VaultsReport, WqReport, ) -from src.modules.submodules.consensus import ( - ConsensusModule, +from src.modules.oracles.common.consensus import ( InitialEpochIsYetToArriveRevert, ) -from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.common.types import OracleModule +from src.modules.common.types import ZERO_HASH, ModuleExecuteDelay from src.providers.execution.contracts.accounting_oracle import AccountingOracleContract from src.services.bunker import BunkerService from src.services.staking_vaults import StakingVaultsService @@ -61,7 +60,7 @@ logger = logging.getLogger(__name__) -class Accounting(BaseModule, ConsensusModule): +class Accounting(OracleModule): """ Accounting module updates the protocol TVL, distributes node-operator rewards, and processes user withdrawal requests. @@ -87,6 +86,9 @@ def __init__(self, w3: Web3): def refresh_contracts(self): self.report_contract = self.w3.lido_contracts.accounting_oracle # type: ignore + def is_contracts_addresses_changed(self) -> bool: + return self.w3.lido_contracts.has_contract_address_changed() + def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecuteDelay: report_blockstamp = self.get_blockstamp_for_report(last_finalized_blockstamp) diff --git a/src/modules/oracles/accounting/entrypoint.py b/src/modules/oracles/accounting/entrypoint.py new file mode 100644 index 000000000..38df6c69f --- /dev/null +++ b/src/modules/oracles/accounting/entrypoint.py @@ -0,0 +1,12 @@ +from src.modules.oracles.accounting.accounting import Accounting +from src.modules.oracles.common.runtime import OracleWeb3Config, build_oracle_web3, run_oracle_module +from src.runtime import log_startup, start_observability +from src.types import OracleModule + + +def run() -> None: + log_startup(OracleModule.ACCOUNTING) + start_observability() + + web3 = build_oracle_web3(OracleWeb3Config(use_ipfs=True)) + run_oracle_module(Accounting(web3)) diff --git a/src/modules/accounting/events.py b/src/modules/oracles/accounting/events.py similarity index 100% rename from src/modules/accounting/events.py rename to src/modules/oracles/accounting/events.py diff --git a/src/modules/csm/__init__.py b/src/modules/oracles/accounting/third_phase/__init__.py similarity index 100% rename from src/modules/csm/__init__.py rename to src/modules/oracles/accounting/third_phase/__init__.py diff --git a/src/modules/accounting/third_phase/extra_data.py b/src/modules/oracles/accounting/third_phase/extra_data.py similarity index 96% rename from src/modules/accounting/third_phase/extra_data.py rename to src/modules/oracles/accounting/third_phase/extra_data.py index 344bd56af..ab38f65c0 100644 --- a/src/modules/accounting/third_phase/extra_data.py +++ b/src/modules/oracles/accounting/third_phase/extra_data.py @@ -2,8 +2,8 @@ from itertools import groupby, batched from typing import Sequence -from src.modules.accounting.third_phase.types import ExtraData, ItemType, ExtraDataLengths, FormatList -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.accounting.third_phase.types import ExtraData, ItemType, ExtraDataLengths, FormatList +from src.modules.common.types import ZERO_HASH from src.types import NodeOperatorGlobalIndex from src.web3py.types import Web3 diff --git a/src/modules/accounting/third_phase/types.py b/src/modules/oracles/accounting/third_phase/types.py similarity index 100% rename from src/modules/accounting/third_phase/types.py rename to src/modules/oracles/accounting/third_phase/types.py diff --git a/src/modules/accounting/types.py b/src/modules/oracles/accounting/types.py similarity index 100% rename from src/modules/accounting/types.py rename to src/modules/oracles/accounting/types.py diff --git a/src/modules/csm/helpers/__init__.py b/src/modules/oracles/common/__init__.py similarity index 100% rename from src/modules/csm/helpers/__init__.py rename to src/modules/oracles/common/__init__.py diff --git a/src/modules/submodules/consensus.py b/src/modules/oracles/common/consensus.py similarity index 99% rename from src/modules/submodules/consensus.py rename to src/modules/oracles/common/consensus.py index 276524d53..d4920bf15 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/oracles/common/consensus.py @@ -20,12 +20,12 @@ ORACLE_MEMBER_INFO, ORACLE_MEMBER_LAST_REPORT_REF_SLOT, ) -from src.modules.submodules.exceptions import ( +from src.modules.oracles.common.exceptions import ( ContractVersionMismatch, IncompatibleOracleVersion, IsNotMemberException, ) -from src.modules.submodules.types import ( +from src.modules.common.types import ( ZERO_HASH, ChainConfig, ConsensusGenesisConfig, diff --git a/src/modules/submodules/exceptions.py b/src/modules/oracles/common/exceptions.py similarity index 100% rename from src/modules/submodules/exceptions.py rename to src/modules/oracles/common/exceptions.py diff --git a/src/modules/submodules/oracle_module.py b/src/modules/oracles/common/oracle_module.py similarity index 50% rename from src/modules/submodules/oracle_module.py rename to src/modules/oracles/common/oracle_module.py index 6ad3d4cc2..ad8d5f655 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/oracles/common/oracle_module.py @@ -1,41 +1,31 @@ import logging -import signal -import time import traceback from abc import abstractmethod, ABC -from dataclasses import asdict -from enum import Enum +from contextlib import contextmanager +from typing import Iterator from requests.exceptions import ConnectionError as RequestsConnectionError -from timeout_decorator import timeout, TimeoutError as DecoratorTimeoutError +from timeout_decorator import TimeoutError as DecoratorTimeoutError from web3.exceptions import Web3Exception -from src.metrics.healthcheck_server import pulse -from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch +from src.modules.common.daemon_module import DaemonModule +from src.modules.common.types import ModuleExecuteDelay +from src.modules.oracles.common.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch from src.providers.http_provider import NotOkResponse from src.providers.ipfs import IPFSError from src.providers.keys.client import KAPIInconsistentData, KeysOutdatedException from src.utils.cache import clear_global_cache from src.web3py.extensions.lido_validators import CountOfKeysDiffersException -from src.utils.blockstamp import build_blockstamp from src.utils.slot import NoSlotsAvailable, SlotNotFinalized, InconsistentData from src.web3py.types import Web3 from web3_multi_provider import NoActiveProviderError -from src import variables -from src.types import SlotNumber, BlockStamp, BlockRoot +from src.types import BlockStamp logger = logging.getLogger(__name__) -class ModuleExecuteDelay(Enum): - """Signals from execute_module method""" - NEXT_SLOT = 0 - NEXT_FINALIZED_EPOCH = 1 - - -class BaseModule(ABC): +class BaseModule(DaemonModule, ABC): """ Base skeleton for Oracle modules. @@ -45,44 +35,26 @@ class BaseModule(ABC): - Check Module didn't stick inside cycle forever. """ - # This is reference mark for long sleep. Sleep until new finalized slot found. - _slot_threshold = SlotNumber(0) - def __init__(self, w3: Web3): self.w3 = w3 - def run_as_daemon(self): - logger.info({'msg': 'Run module as daemon.'}) - while True: - logger.debug({'msg': 'Startup new cycle.'}) - self.cycle_handler() + def _get_consensus_client(self): + """Returns consensus client from w3""" + return self.w3.cc - def cycle_handler(self): - self._cycle() - self._sleep_cycle() + def run_cycle(self, last_finalized_blockstamp: BlockStamp): + """Override to add Oracle-specific logic before execution""" + self.refresh_contracts_if_address_change() + super().run_cycle(last_finalized_blockstamp) - @timeout(variables.MAX_CYCLE_LIFETIME_IN_SECONDS) - def _cycle(self): - """ - Main cycle logic: fetch the last finalized slot, refresh contracts if necessary, - and execute the module's business logic. - """ + @contextmanager + def exception_handler(self) -> Iterator[None]: # pylint: disable=too-many-branches + """Context manager for handling Oracle module cycle exceptions""" try: - blockstamp = self._receive_last_finalized_slot() - - # Check if the blockstamp is below the threshold and exit early - if blockstamp.slot_number <= self._slot_threshold: - logger.info({ - 'msg': 'Skipping the report. Waiting for new finalized slot.', - 'slot_threshold': self._slot_threshold, - }) - return - - self.refresh_contracts_if_address_change() - self.run_cycle(blockstamp) + yield except IsNotMemberException as error: - logger.error({'msg': 'Provided account is not part of Oracle`s committee.'}) + logger.error({'msg': 'Provided account is not part of Oracle\'s committee.'}) raise error except IncompatibleOracleVersion as error: logger.error({'msg': 'Incompatible Contract version. Please update Oracle Daemon.'}) @@ -114,35 +86,8 @@ def _cycle(self): logger.error({'msg': 'IPFS provider error.', 'error': str(error)}) except ValueError as error: logger.error({'msg': 'Unexpected error.', 'error': str(error)}) - - @staticmethod - def _reset_cycle_timeout(): - """Reset the timeout timer for the current cycle.""" - logger.info({'msg': f'Reset running cycle timeout to {variables.MAX_CYCLE_LIFETIME_IN_SECONDS} seconds'}) - signal.setitimer(signal.ITIMER_REAL, variables.MAX_CYCLE_LIFETIME_IN_SECONDS) - pulse() - - @staticmethod - def _sleep_cycle(): - """Handles sleeping between cycles based on the configured cycle sleep time.""" - logger.info({'msg': f'Cycle end. Sleeping for {variables.CYCLE_SLEEP_IN_SECONDS} seconds.'}) - time.sleep(variables.CYCLE_SLEEP_IN_SECONDS) - - def _receive_last_finalized_slot(self) -> BlockStamp: - block_root = BlockRoot(self.w3.cc.get_block_root('finalized').root) - block_details = self.w3.cc.get_block_details(block_root) - bs = build_blockstamp(block_details) - logger.info({'msg': 'Fetch last finalized BlockStamp.', 'value': asdict(bs)}) - ORACLE_SLOT_NUMBER.labels('finalized').set(bs.slot_number) - ORACLE_BLOCK_NUMBER.labels('finalized').set(bs.block_number) - return bs - - def run_cycle(self, blockstamp: BlockStamp): - logger.info({'msg': 'Execute module.', 'value': blockstamp}) - result = self.execute_module(blockstamp) - pulse() - if result is ModuleExecuteDelay.NEXT_FINALIZED_EPOCH: - self._slot_threshold = blockstamp.slot_number + except Exception as error: + raise error @abstractmethod def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecuteDelay: @@ -152,15 +97,16 @@ def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecute ModuleExecuteDelay.NEXT_FINALIZED_EPOCH - to sleep until new finalized epoch ModuleExecuteDelay.NEXT_SLOT - to sleep for a slot """ - raise NotImplementedError('Module should implement this method.') # pragma: no cover @abstractmethod def refresh_contracts(self): """This method called if contracts addresses were changed""" - raise NotImplementedError('Module should implement this method.') # pragma: no cover + + @abstractmethod + def is_contracts_addresses_changed(self) -> bool: + """Return True if underlying contracts addresses changed and refresh is needed.""" def refresh_contracts_if_address_change(self): - # Refresh contracts if the address has changed - if self.w3.lido_contracts.has_contract_address_changed(): + if self.is_contracts_addresses_changed(): clear_global_cache() self.refresh_contracts() diff --git a/src/modules/oracles/common/runtime.py b/src/modules/oracles/common/runtime.py new file mode 100644 index 000000000..19cc5cc76 --- /dev/null +++ b/src/modules/oracles/common/runtime.py @@ -0,0 +1,151 @@ +from dataclasses import dataclass +from typing import Iterator, cast, Any + +from packaging.version import Version +from web3_multi_provider.metrics import init_metrics + +from src import constants, variables +from src.metrics.logging import logging +from src.modules.oracles.common.types import OracleModule +from src.providers.ipfs import IPFSProvider, Kubo, LidoIPFS, Pinata, Storacha +from src.utils.exception import IncompatibleException +from src.web3py.contract_tweak import tweak_w3_contracts +from src.web3py.extensions import ( + ConsensusClientModule, + FallbackProviderModule, + IPFS, + KeysAPIClientModule, + LidoContracts, + LidoValidatorsProvider, + TransactionUtils, +) +from src.web3py.extensions.staking_module import StakingModuleContracts +from src.web3py.extensions.performance import PerformanceClientModule +from src.web3py.types import Web3 + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class OracleWeb3Config: + use_ipfs: bool = False + use_performance_client: bool = False + use_staking_module_contracts: bool = False + use_lido_contracts: bool = True + use_lido_validators: bool = True + + +def build_oracle_web3(config: OracleWeb3Config) -> Web3: + logger.info({'msg': 'Initialize multi web3 provider.'}) + web3 = Web3(FallbackProviderModule( + variables.EXECUTION_CLIENT_URI, + request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION}, + cache_allowed_requests=True, + )) + + logger.info({'msg': 'Modify web3 with custom contract function call.'}) + tweak_w3_contracts(web3) + + logger.info({'msg': 'Initialize consensus client.'}) + cc = ConsensusClientModule(variables.CONSENSUS_CLIENT_URI, web3) + + logger.info({'msg': 'Initialize keys api client.'}) + kac = KeysAPIClientModule(variables.KEYS_API_URI, web3) + + logger.info({'msg': 'Check configured providers.'}) + if Version(kac.get_status().appVersion) < constants.ALLOWED_KAPI_VERSION: + raise IncompatibleException(f'Incompatible KAPI version. Required >= {constants.ALLOWED_KAPI_VERSION}.') + + check_providers_chain_ids(web3, cc, kac) + + modules: dict[str, Any] = { + 'transaction': TransactionUtils, + 'cc': lambda: cc, + 'kac': lambda: kac, + } + + if config.use_lido_contracts: + modules['lido_contracts'] = LidoContracts + + if config.use_lido_validators: + modules['lido_validators'] = LidoValidatorsProvider + + if config.use_staking_module_contracts: + modules['staking_module'] = StakingModuleContracts + + if config.use_ipfs: + ipfs = IPFS(web3, ipfs_providers(), retries=variables.HTTP_REQUEST_RETRY_COUNT_IPFS) + modules['ipfs'] = lambda: ipfs + + if config.use_performance_client: + performance = PerformanceClientModule(variables.PERFORMANCE_COLLECTOR_URI) + modules['performance'] = lambda: performance + + web3.attach_modules(modules) + + logger.info({'msg': 'Initialize prometheus metrics.'}) + init_metrics() + + return web3 + + +def run_oracle_module(module: OracleModule): + module.check_contract_configs() + + if variables.DAEMON: + module.run_as_daemon() + else: + module.cycle_handler() + + +def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): + keys_api_chain_id = kac.check_providers_consistency() + consensus_chain_id = cc.check_providers_consistency() + execution_chain_id = cast(FallbackProviderModule, web3.provider).check_providers_consistency() + + if execution_chain_id == consensus_chain_id == keys_api_chain_id: + return + + raise IncompatibleException( + 'Different chain ids detected:\n' + f'Execution chain id: {execution_chain_id}\n' + f'Consensus chain id: {consensus_chain_id}\n' + f'Keys API chain id: {keys_api_chain_id}\n' + ) + + +def ipfs_providers() -> Iterator[IPFSProvider]: + if variables.KUBO_HOST: + yield Kubo( + variables.KUBO_HOST, + variables.KUBO_RPC_PORT, + variables.KUBO_GATEWAY_PORT, + timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, + ) + + if variables.PINATA_JWT and variables.PINATA_DEDICATED_GATEWAY_URL and variables.PINATA_DEDICATED_GATEWAY_TOKEN: + yield Pinata( + variables.PINATA_JWT, + timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, + dedicated_gateway_url=variables.PINATA_DEDICATED_GATEWAY_URL, + dedicated_gateway_token=variables.PINATA_DEDICATED_GATEWAY_TOKEN, + ) + + if ( + variables.STORACHA_AUTH_SECRET and + variables.STORACHA_AUTHORIZATION and + variables.STORACHA_SPACE_DID + ): + yield Storacha( + variables.STORACHA_AUTH_SECRET, + variables.STORACHA_AUTHORIZATION, + variables.STORACHA_SPACE_DID, + timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, + ) + + if variables.LIDO_IPFS_HOST and variables.LIDO_IPFS_TOKEN: + yield LidoIPFS( + variables.LIDO_IPFS_HOST, + variables.LIDO_IPFS_TOKEN, + timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, + ) diff --git a/src/modules/oracles/common/types.py b/src/modules/oracles/common/types.py new file mode 100644 index 000000000..0e338287c --- /dev/null +++ b/src/modules/oracles/common/types.py @@ -0,0 +1,8 @@ +from abc import ABC + +from src.modules.oracles.common.consensus import ConsensusModule +from src.modules.oracles.common.oracle_module import BaseModule + + +class OracleModule(BaseModule, ConsensusModule, ABC): + pass diff --git a/src/modules/ejector/__init__.py b/src/modules/oracles/ejector/__init__.py similarity index 100% rename from src/modules/ejector/__init__.py rename to src/modules/oracles/ejector/__init__.py diff --git a/src/modules/ejector/data_encode.py b/src/modules/oracles/ejector/data_encode.py similarity index 100% rename from src/modules/ejector/data_encode.py rename to src/modules/oracles/ejector/data_encode.py diff --git a/src/modules/ejector/ejector.py b/src/modules/oracles/ejector/ejector.py similarity index 95% rename from src/modules/ejector/ejector.py rename to src/modules/oracles/ejector/ejector.py index 35c5fe804..8f2222cc7 100644 --- a/src/modules/ejector/ejector.py +++ b/src/modules/oracles/ejector/ejector.py @@ -15,12 +15,12 @@ EJECTOR_TO_WITHDRAW_WEI_AMOUNT, EJECTOR_VALIDATORS_COUNT_TO_EJECT, ) -from src.modules.ejector.data_encode import encode_data -from src.modules.ejector.sweep import get_sweep_delay_in_epochs -from src.modules.ejector.types import EjectorProcessingState, ReportData -from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert -from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.ejector.data_encode import encode_data +from src.modules.oracles.ejector.sweep import get_sweep_delay_in_epochs +from src.modules.oracles.ejector.types import EjectorProcessingState, ReportData +from src.modules.oracles.common.consensus import InitialEpochIsYetToArriveRevert +from src.modules.oracles.common.types import OracleModule +from src.modules.common.types import ZERO_HASH, ModuleExecuteDelay from src.providers.consensus.types import Validator, BeaconStateView from src.providers.execution.contracts.exit_bus_oracle import ExitBusOracleContract from src.services.exit_order_iterator import ValidatorExitIterator @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) -class Ejector(BaseModule, ConsensusModule): +class Ejector(OracleModule): """ Module that ejects lido validators depends on total value of unfinalized withdrawal requests. @@ -74,6 +74,9 @@ def __init__(self, w3: Web3): def refresh_contracts(self): self.report_contract = self.w3.lido_contracts.validators_exit_bus_oracle # type: ignore + def is_contracts_addresses_changed(self) -> bool: + return self.w3.lido_contracts.has_contract_address_changed() + def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecuteDelay: report_blockstamp = self.get_blockstamp_for_report(last_finalized_blockstamp) diff --git a/src/modules/oracles/ejector/entrypoint.py b/src/modules/oracles/ejector/entrypoint.py new file mode 100644 index 000000000..30347576b --- /dev/null +++ b/src/modules/oracles/ejector/entrypoint.py @@ -0,0 +1,12 @@ +from src.modules.oracles.common.runtime import OracleWeb3Config, build_oracle_web3, run_oracle_module +from src.modules.oracles.ejector.ejector import Ejector +from src.runtime import log_startup, start_observability +from src.types import OracleModule + + +def run() -> None: + log_startup(OracleModule.EJECTOR) + start_observability() + + web3 = build_oracle_web3(OracleWeb3Config()) + run_oracle_module(Ejector(web3)) diff --git a/src/modules/ejector/sweep.py b/src/modules/oracles/ejector/sweep.py similarity index 99% rename from src/modules/ejector/sweep.py rename to src/modules/oracles/ejector/sweep.py index f8c370296..d60049d04 100644 --- a/src/modules/ejector/sweep.py +++ b/src/modules/oracles/ejector/sweep.py @@ -9,7 +9,7 @@ MAX_WITHDRAWALS_PER_PAYLOAD, MIN_ACTIVATION_BALANCE, ) -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import BeaconStateView from src.types import Gwei from src.utils.validator_state import ( diff --git a/src/modules/ejector/types.py b/src/modules/oracles/ejector/types.py similarity index 100% rename from src/modules/ejector/types.py rename to src/modules/oracles/ejector/types.py diff --git a/src/modules/performance/__init__.py b/src/modules/oracles/staking_modules/__init__.py similarity index 100% rename from src/modules/performance/__init__.py rename to src/modules/oracles/staking_modules/__init__.py diff --git a/src/modules/csm/csm.py b/src/modules/oracles/staking_modules/base.py similarity index 84% rename from src/modules/csm/csm.py rename to src/modules/oracles/staking_modules/base.py index dbcb2543e..a340ba16c 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/oracles/staking_modules/base.py @@ -5,20 +5,19 @@ from src.constants import UINT64_MAX from src.metrics.prometheus.business import CONTRACT_ON_PAUSE -from src.metrics.prometheus.csm import ( - CSM_CURRENT_FRAME_RANGE_L_EPOCH, - CSM_CURRENT_FRAME_RANGE_R_EPOCH, +from src.metrics.prometheus.staking_module import ( + STAKING_MODULE_CURRENT_FRAME_RANGE_L_EPOCH, + STAKING_MODULE_CURRENT_FRAME_RANGE_R_EPOCH, ) from src.metrics.prometheus.duration_meter import duration_meter -from src.modules.csm.distribution import Distribution, DistributionResult, StrikesValidator -from src.modules.csm.helpers.last_report import LastReport -from src.modules.csm.log import Logs -from src.modules.csm.state import State -from src.modules.csm.tree import RewardsTree, StrikesTree, Tree -from src.modules.csm.types import ReportData, RewardsShares, StrikesList -from src.modules.submodules.consensus import ConsensusModule -from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.staking_modules.common.distribution import Distribution, DistributionResult +from src.modules.oracles.staking_modules.common.helpers.last_report import LastReport +from src.modules.oracles.staking_modules.common.log import Logs +from src.modules.oracles.staking_modules.common.state import State +from src.modules.oracles.staking_modules.common.tree import RewardsTree, StrikesTree, Tree +from src.modules.oracles.staking_modules.common.types import ReportData, RewardsShares, StrikesList, StrikesValidator +from src.modules.oracles.common.types import OracleModule +from src.modules.common.types import ZERO_HASH, ModuleExecuteDelay from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract from src.providers.execution.exceptions import InconsistentData from src.providers.ipfs import CID @@ -39,37 +38,48 @@ logger = logging.getLogger(__name__) -class CSMError(Exception): - """Unrecoverable error in CSM module""" +class SMPerformanceOracleError(Exception): + """Unrecoverable error in staking module performance oracle""" -class CSOracle(BaseModule, ConsensusModule): +class SMPerformanceOracle(OracleModule): """ - CSM performance module collects performance of CSM node operators and creates a Merkle tree of the resulting - distribution of shares among the operators. The root of the tree is then submitted to the module contract. + Staking Module performance oracle collects performance of staking module node operators and creates a Merkle tree + of the resulting distribution of shares among the operators. The root of the tree is then submitted to the + module contract. The algorithm for calculating performance includes the following steps: 1. Collect all the attestation duties of the network validators for the frame. 2. Calculate the performance of each validator based on the attestations. - 3. Calculate the share of each CSM node operator excluding underperforming validators. + 3. Calculate the share of each node operator excluding underperforming validators. """ - COMPATIBLE_CONTRACT_VERSION = 2 - COMPATIBLE_CONSENSUS_VERSION = 3 + COMPATIBLE_CONTRACT_VERSION: int = 0 + COMPATIBLE_CONSENSUS_VERSION: int = 0 report_contract: CSFeeOracleContract + state: State def __init__(self, w3: Web3): + if self.COMPATIBLE_CONTRACT_VERSION == 0: + raise ValueError("CONTRACT_VERSION is not defined") + if self.COMPATIBLE_CONSENSUS_VERSION == 0: + raise ValueError("CONSENSUS_VERSION is not defined") self.consumer = self.__class__.__name__ - self.report_contract = w3.csm.oracle - self.state = State.load() + self.report_contract = w3.staking_module.oracle + self.state = State.load(self.consumer) super().__init__(w3) atexit.register(self._on_shutdown) def refresh_contracts(self): - self.report_contract = self.w3.csm.oracle + self.report_contract = self.w3.staking_module.oracle + self.w3.staking_module.reload_contracts() + self.report_contract = self.w3.staking_module.oracle # type: ignore self.state.clear() + def is_contracts_addresses_changed(self) -> bool: + return self.w3.staking_module.has_contract_address_changed() + def _on_shutdown(self): performance_client = getattr(self.w3, "performance", None) if performance_client is None: @@ -167,11 +177,9 @@ def collect_data(self) -> bool: def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: self.validate_state(blockstamp) - last_report = self._get_last_report(blockstamp) + distribution, last_report = self.calculate_distribution(blockstamp) rewards_tree_root, rewards_cid = last_report.rewards_tree_root, last_report.rewards_tree_cid - distribution = self.calculate_distribution(blockstamp, last_report) - if distribution.total_rewards: rewards_tree = self.make_rewards_tree(distribution.total_rewards_map) rewards_tree_root = rewards_tree.root @@ -208,13 +216,14 @@ def _get_last_report(self, blockstamp: ReferenceBlockStamp) -> LastReport: return LastReport.load(self.w3, blockstamp, current_frame) @lru_cache(maxsize=1) - def calculate_distribution(self, blockstamp: ReferenceBlockStamp, last_report: LastReport) -> DistributionResult: + def calculate_distribution(self, blockstamp: ReferenceBlockStamp) -> tuple[DistributionResult, LastReport]: + last_report = self._get_last_report(blockstamp) distribution = Distribution(self.w3, self.converter(blockstamp), self.state) result = distribution.calculate(blockstamp, last_report) - return result + return result, last_report def is_main_data_submitted(self, blockstamp: BlockStamp) -> bool: - last_ref_slot = self.w3.csm.get_csm_last_processing_ref_slot(blockstamp) + last_ref_slot = self.w3.staking_module.get_last_processing_ref_slot(blockstamp) ref_slot = self.get_initial_or_current_frame(blockstamp).ref_slot return last_ref_slot == ref_slot @@ -329,7 +338,7 @@ def make_rewards_tree(self, shares: dict[NodeOperatorId, RewardsShares]) -> Rewa # XXX: We put a stone here to make sure, that even with only 1 node operator in the tree, it's still possible to # claim rewards. The CSModule contract skips pulling rewards if the proof's length is zero, which is the case # when the tree has only one leaf. - stone = NodeOperatorId(self.w3.csm.module.MAX_OPERATORS_COUNT) + stone = NodeOperatorId(self.w3.staking_module.module.MAX_OPERATORS_COUNT) shares[stone] = 0 # XXX: Remove the stone as soon as we have enough leafs to build a suitable tree. @@ -363,9 +372,9 @@ def get_epochs_range_to_process(self, blockstamp: BlockStamp) -> tuple[EpochNumb far_future_initial_epoch = converter.get_epoch_by_timestamp(UINT64_MAX) if converter.frame_config.initial_epoch == far_future_initial_epoch: - raise ValueError("CSM oracle initial epoch is not set yet") + raise ValueError("Oracle initial epoch is not set yet") - l_ref_slot = last_processing_ref_slot = self.w3.csm.get_csm_last_processing_ref_slot(blockstamp) + l_ref_slot = last_processing_ref_slot = self.w3.staking_module.get_last_processing_ref_slot(blockstamp) r_ref_slot = initial_ref_slot = self.get_initial_ref_slot(blockstamp) if last_processing_ref_slot > blockstamp.slot_number: @@ -375,7 +384,7 @@ def get_epochs_range_to_process(self, blockstamp: BlockStamp) -> tuple[EpochNumb if not last_processing_ref_slot: l_ref_slot = SlotNumber(initial_ref_slot - converter.slots_per_frame) if l_ref_slot < 0: - raise CSMError("Invalid frame configuration for the current network") + raise SMPerformanceOracleError("Invalid frame configuration for the current network") # NOTE: before the initial slot the contract can't return current frame if blockstamp.slot_number > initial_ref_slot: @@ -389,16 +398,16 @@ def get_epochs_range_to_process(self, blockstamp: BlockStamp) -> tuple[EpochNumb ) if l_ref_slot < last_processing_ref_slot: - raise CSMError(f"Got invalid epochs range: {l_ref_slot=} < {last_processing_ref_slot=}") + raise SMPerformanceOracleError(f"Got invalid epochs range: {l_ref_slot=} < {last_processing_ref_slot=}") if l_ref_slot >= r_ref_slot: - raise CSMError(f"Got invalid epochs range {r_ref_slot=}, {l_ref_slot=}") + raise SMPerformanceOracleError(f"Got invalid epochs range {r_ref_slot=}, {l_ref_slot=}") l_epoch = converter.get_epoch_by_slot(SlotNumber(l_ref_slot + 1)) r_epoch = converter.get_epoch_by_slot(r_ref_slot) # Update Prometheus metrics - CSM_CURRENT_FRAME_RANGE_L_EPOCH.set(l_epoch) - CSM_CURRENT_FRAME_RANGE_R_EPOCH.set(r_epoch) + STAKING_MODULE_CURRENT_FRAME_RANGE_L_EPOCH.set(l_epoch) + STAKING_MODULE_CURRENT_FRAME_RANGE_R_EPOCH.set(r_epoch) logger.info({ "msg": "Epochs range for the report", diff --git a/src/modules/performance/collector/__init__.py b/src/modules/oracles/staking_modules/common/__init__.py similarity index 100% rename from src/modules/performance/collector/__init__.py rename to src/modules/oracles/staking_modules/common/__init__.py diff --git a/src/modules/csm/distribution.py b/src/modules/oracles/staking_modules/common/distribution.py similarity index 95% rename from src/modules/csm/distribution.py rename to src/modules/oracles/staking_modules/common/distribution.py index 7e140aa45..05640b8ef 100644 --- a/src/modules/csm/distribution.py +++ b/src/modules/oracles/staking_modules/common/distribution.py @@ -5,10 +5,10 @@ from dataclasses import dataclass, field from src.constants import MIN_ACTIVATION_BALANCE, MAX_EFFECTIVE_BALANCE_ELECTRA, EFFECTIVE_BALANCE_INCREMENT -from src.modules.csm.helpers.last_report import LastReport -from src.modules.csm.log import FramePerfLog, OperatorFrameSummary, Logs -from src.modules.csm.state import Frame, State, ValidatorDuties -from src.modules.csm.types import ( +from src.modules.oracles.staking_modules.common.helpers.last_report import LastReport +from src.modules.oracles.staking_modules.common.log import FramePerfLog, OperatorFrameSummary, Logs +from src.modules.oracles.staking_modules.common.state import Frame, State, ValidatorDuties +from src.modules.oracles.staking_modules.common.types import ( ParticipationShares, RewardsShares, StrikesList, @@ -75,7 +75,7 @@ def calculate(self, blockstamp: ReferenceBlockStamp, last_report: LastReport) -> frame_blockstamp = self._get_frame_blockstamp(blockstamp, to_epoch) frame_module_validators = self._get_module_validators(frame_blockstamp) - total_rewards_to_distribute = self.w3.csm.fee_distributor.shares_to_distribute(frame_blockstamp.block_hash) + total_rewards_to_distribute = self.w3.staking_module.fee_distributor.shares_to_distribute(frame_blockstamp.block_hash) rewards_to_distribute_in_frame = total_rewards_to_distribute - distributed_so_far frame_log = FramePerfLog(frame_blockstamp, frame) @@ -129,7 +129,8 @@ def _get_ref_blockstamp_for_frame( def _get_module_validators(self, blockstamp: ReferenceBlockStamp) -> ValidatorsByNodeOperator: return self.w3.lido_validators.get_used_module_validators_by_node_operators( - StakingModuleAddress(self.w3.csm.module.address), blockstamp + StakingModuleAddress(self.w3.staking_module.module.address), + blockstamp, ) def _calculate_distribution_in_frame( @@ -155,7 +156,7 @@ def _calculate_distribution_in_frame( logger.info({"msg": f"Calculating distribution for {no_id=}"}) log_operator = log.operators[no_id] - curve_params = self.w3.csm.get_curve_params(no_id, blockstamp) + curve_params = self.w3.staking_module.get_curve_params(no_id, blockstamp) log_operator.performance_coefficients = curve_params.perf_coeffs # Sort from biggest to smallest balance and by index from oldest to newest. @@ -339,7 +340,7 @@ def _process_strikes( no_id, _ = key if key not in strikes_in_frame: merged[key].push(StrikesList.SENTINEL) # Just shifting... - maxlen = self.w3.csm.get_curve_params(no_id, frame_blockstamp).strikes_params.lifetime + maxlen = self.w3.staking_module.get_curve_params(no_id, frame_blockstamp).strikes_params.lifetime merged[key].resize(maxlen) # NOTE: Cleanup sequences like [0,0,0] since they don't bring any information. if not sum(merged[key]): diff --git a/src/modules/performance/common/__init__.py b/src/modules/oracles/staking_modules/common/helpers/__init__.py similarity index 100% rename from src/modules/performance/common/__init__.py rename to src/modules/oracles/staking_modules/common/helpers/__init__.py diff --git a/src/modules/csm/helpers/last_report.py b/src/modules/oracles/staking_modules/common/helpers/last_report.py similarity index 83% rename from src/modules/csm/helpers/last_report.py rename to src/modules/oracles/staking_modules/common/helpers/last_report.py index eea09ff0a..401c6da1e 100644 --- a/src/modules/csm/helpers/last_report.py +++ b/src/modules/oracles/staking_modules/common/helpers/last_report.py @@ -5,12 +5,12 @@ from hexbytes import HexBytes -from src.modules.csm.tree import RewardsTree, StrikesTree -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.staking_modules.common.tree import RewardsTree, StrikesTree +from src.modules.common.types import ZERO_HASH from src.providers.execution.exceptions import InconsistentData from src.providers.ipfs import CID from src.types import BlockStamp, FrameNumber -from src.modules.csm.types import RewardsTreeLeaf, StrikesList, StrikesValidator +from src.modules.oracles.staking_modules.common.types import RewardsTreeLeaf, StrikesList, StrikesValidator from src.web3py.types import Web3 logger = logging.getLogger(__name__) @@ -29,14 +29,14 @@ class LastReport: @classmethod def load(cls, w3: Web3, blockstamp: BlockStamp, current_frame: FrameNumber) -> Self: - rewards_tree_root = w3.csm.get_rewards_tree_root(blockstamp) - rewards_tree_cid = w3.csm.get_rewards_tree_cid(blockstamp) + rewards_tree_root = w3.staking_module.get_rewards_tree_root(blockstamp) + rewards_tree_cid = w3.staking_module.get_rewards_tree_cid(blockstamp) if (rewards_tree_cid is None) != (rewards_tree_root == ZERO_HASH): raise InconsistentData(f"Got inconsistent previous tree data: {rewards_tree_root=} {rewards_tree_cid=}") - strikes_tree_root = w3.csm.get_strikes_tree_root(blockstamp) - strikes_tree_cid = w3.csm.get_strikes_tree_cid(blockstamp) + strikes_tree_root = w3.staking_module.get_strikes_tree_root(blockstamp) + strikes_tree_cid = w3.staking_module.get_strikes_tree_cid(blockstamp) if (strikes_tree_cid is None) != (strikes_tree_root == ZERO_HASH): raise InconsistentData(f"Got inconsistent previous tree data: {strikes_tree_root=} {strikes_tree_cid=}") diff --git a/src/modules/csm/log.py b/src/modules/oracles/staking_modules/common/log.py similarity index 88% rename from src/modules/csm/log.py rename to src/modules/oracles/staking_modules/common/log.py index 745de0aaa..418e5d278 100644 --- a/src/modules/csm/log.py +++ b/src/modules/oracles/staking_modules/common/log.py @@ -2,9 +2,9 @@ from collections import defaultdict from dataclasses import asdict, dataclass, field -from src.constants import CSM_LOGS_VERSION -from src.modules.csm.state import DutyAccumulator -from src.modules.csm.types import RewardsShares +from src.constants import STAKING_MODULE_LOGS_VERSION +from src.modules.oracles.staking_modules.common.state import DutyAccumulator +from src.modules.oracles.staking_modules.common.types import RewardsShares from src.providers.execution.contracts.cs_parameters_registry import PerformanceCoefficients from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex @@ -48,7 +48,7 @@ class FramePerfLog: @dataclass class Logs: frames: list[FramePerfLog] = field(default_factory=list) - _ver: int = CSM_LOGS_VERSION + _ver: int = STAKING_MODULE_LOGS_VERSION def encode(self) -> bytes: return ( diff --git a/src/modules/csm/state.py b/src/modules/oracles/staking_modules/common/state.py similarity index 87% rename from src/modules/csm/state.py rename to src/modules/oracles/staking_modules/common/state.py index 6f183f4b2..747518cf2 100644 --- a/src/modules/csm/state.py +++ b/src/modules/oracles/staking_modules/common/state.py @@ -9,7 +9,7 @@ from typing import Self from src import variables -from src.constants import CSM_STATE_VERSION +from src.constants import STAKING_MODULE_STATE_VERSION from src.types import EpochNumber, ValidatorIndex from src.utils.range import sequence @@ -71,9 +71,9 @@ class State: # pylint: disable=too-many-public-methods """ - Processing state of a CSM performance oracle frame. + Processing state of a staking module performance oracle frame. - During the CSM module startup the state object is being either `load`'ed from the filesystem or being created as a + During the module startup the state object is being either `load`'ed from the filesystem or being created as a new object with no data in it. During epochs processing aggregates in `data` are being updated and eventually the state is `commit`'ed back to the filesystem. @@ -86,50 +86,65 @@ class State: _processed_epochs: set[EpochNumber] _version: int + _oracle_name: str EXTENSION = ".pkl" - def __init__(self) -> None: + def __init__(self, oracle_name: str) -> None: self.data = {} self._epochs_to_process = tuple() self._processed_epochs = set() - self._version = CSM_STATE_VERSION + self._version = STAKING_MODULE_STATE_VERSION + self._oracle_name = oracle_name @property def version(self) -> int | None: return getattr(self, "_version", None) + @property + def oracle_name(self) -> str: + return getattr(self, "_oracle_name", "unknown") + @classmethod - def load(cls) -> Self: + def load(cls, oracle_name: str) -> Self: """Used to restore the object from the persistent storage""" obj: Self | None = None - file = cls.file() + file = cls.file(oracle_name) try: with file.open(mode="rb") as f: obj = pickle.load(f) print({"msg": "Read object from pickle file"}) if not obj: raise ValueError("Got empty object") + # Ensure loaded object has the correct oracle_name + # pylint: disable=protected-access + if obj._oracle_name != oracle_name: + logger.warning({ + "msg": f"Cache oracle name mismatch: {obj._oracle_name} != {oracle_name}. Creating new state." + }) + return cls(oracle_name) + # Set oracle_name for backward compatibility with old cache files + obj._oracle_name = oracle_name except Exception as e: # pylint: disable=broad-exception-caught logger.info({"msg": f"Unable to restore {cls.__name__} instance from {file.absolute()}", "error": str(e)}) else: logger.info({"msg": f"{cls.__name__} read from {file.absolute()}"}) - return obj or cls() + return obj or cls(oracle_name) def commit(self) -> None: with self.buffer.open(mode="wb") as f: pickle.dump(self, f) - os.replace(self.buffer, self.file()) + os.replace(self.buffer, self.file(self._oracle_name)) @classmethod - def file(cls) -> Path: - return variables.CACHE_PATH / Path("cache").with_suffix(cls.EXTENSION) + def file(cls, oracle_name: str) -> Path: + return variables.CACHE_PATH / Path(f"{oracle_name}_cache").with_suffix(cls.EXTENSION) @property def buffer(self) -> Path: - return self.file().with_suffix(".buf") + return self.file(self._oracle_name).with_suffix(".buf") @property def is_empty(self) -> bool: @@ -167,7 +182,7 @@ def clear(self) -> None: self._processed_epochs.clear() assert self.is_empty - @lru_cache(variables.CSM_ORACLE_MAX_CONCURRENCY) + @lru_cache(maxsize=1) def find_frame(self, epoch: EpochNumber) -> Frame: for epoch_range in self.frames: from_epoch, to_epoch = epoch_range @@ -194,12 +209,12 @@ def log_progress(self) -> None: logger.info({"msg": f"Processed {len(self._processed_epochs)} of {len(self._epochs_to_process)} epochs"}) def migrate(self, l_epoch: EpochNumber, r_epoch: EpochNumber, epochs_per_frame: int) -> None: - if self.version != CSM_STATE_VERSION: + if self.version != STAKING_MODULE_STATE_VERSION: if self.version is not None: logger.warning( { "msg": f"Cache was built with version={self.version}. " - f"Discarding data to migrate to cache version={CSM_STATE_VERSION}" + f"Discarding data to migrate to cache version={STAKING_MODULE_STATE_VERSION}" } ) self.clear() @@ -212,7 +227,7 @@ def migrate(self, l_epoch: EpochNumber, r_epoch: EpochNumber, epochs_per_frame: self.find_frame.cache_clear() self._epochs_to_process = tuple(sequence(l_epoch, r_epoch)) - self._version = CSM_STATE_VERSION + self._version = STAKING_MODULE_STATE_VERSION self.commit() def _migrate_frames_data(self, new_frames: list[Frame]): diff --git a/src/modules/csm/tree.py b/src/modules/oracles/staking_modules/common/tree.py similarity index 97% rename from src/modules/csm/tree.py rename to src/modules/oracles/staking_modules/common/tree.py index 92abb5098..c52ba2d4c 100644 --- a/src/modules/csm/tree.py +++ b/src/modules/oracles/staking_modules/common/tree.py @@ -6,7 +6,7 @@ from hexbytes import HexBytes from oz_merkle_tree import Dump, StandardMerkleTree -from src.modules.csm.types import RewardsTreeLeaf, StrikesList, StrikesTreeLeaf +from src.modules.oracles.staking_modules.common.types import RewardsTreeLeaf, StrikesList, StrikesTreeLeaf from src.utils.types import hex_str_to_bytes @@ -21,7 +21,7 @@ class TreeJSONDecoder(JSONDecoder): ... class Tree[LeafType: Iterable](ABC): - """A wrapper around StandardMerkleTree to cover use cases of the CSM oracle""" + """A wrapper around StandardMerkleTree to cover use cases of the staking module oracle""" encoder: ClassVar[type[JSONEncoder]] = TreeJSONEncoder decoder: ClassVar[type[JSONDecoder]] = TreeJSONDecoder diff --git a/src/modules/csm/types.py b/src/modules/oracles/staking_modules/common/types.py similarity index 100% rename from src/modules/csm/types.py rename to src/modules/oracles/staking_modules/common/types.py diff --git a/src/modules/performance/web/__init__.py b/src/modules/oracles/staking_modules/community_staking/__init__.py similarity index 100% rename from src/modules/performance/web/__init__.py rename to src/modules/oracles/staking_modules/community_staking/__init__.py diff --git a/src/modules/oracles/staking_modules/community_staking/csm.py b/src/modules/oracles/staking_modules/community_staking/csm.py new file mode 100644 index 000000000..df3733fc5 --- /dev/null +++ b/src/modules/oracles/staking_modules/community_staking/csm.py @@ -0,0 +1,8 @@ +from src.modules.oracles.staking_modules.base import SMPerformanceOracle + + +class CSPerformanceOracle(SMPerformanceOracle): + """Community Staking Module Performance Oracle""" + + COMPATIBLE_CONTRACT_VERSION = 2 + COMPATIBLE_CONSENSUS_VERSION = 3 diff --git a/src/modules/oracles/staking_modules/community_staking/entrypoint.py b/src/modules/oracles/staking_modules/community_staking/entrypoint.py new file mode 100644 index 000000000..6da37cb62 --- /dev/null +++ b/src/modules/oracles/staking_modules/community_staking/entrypoint.py @@ -0,0 +1,17 @@ +from src.modules.oracles.common.runtime import OracleWeb3Config, build_oracle_web3, run_oracle_module +from src.modules.oracles.staking_modules.community_staking.csm import CSPerformanceOracle +from src.runtime import log_startup, start_observability +from src.types import OracleModule + + +def run() -> None: + log_startup(OracleModule.CSM) + start_observability() + + web3 = build_oracle_web3(OracleWeb3Config( + use_lido_contracts=False, + use_staking_module_contracts=True, + use_ipfs=True, + use_performance_client=True, + )) + run_oracle_module(CSPerformanceOracle(web3)) diff --git a/src/modules/submodules/__init__.py b/src/modules/oracles/staking_modules/curated/__init__.py similarity index 100% rename from src/modules/submodules/__init__.py rename to src/modules/oracles/staking_modules/curated/__init__.py diff --git a/src/modules/oracles/staking_modules/curated/cm.py b/src/modules/oracles/staking_modules/curated/cm.py new file mode 100644 index 000000000..524bb8209 --- /dev/null +++ b/src/modules/oracles/staking_modules/curated/cm.py @@ -0,0 +1,8 @@ +from src.modules.oracles.staking_modules.base import SMPerformanceOracle + + +class CMPerformanceOracle(SMPerformanceOracle): + """Curated Module Performance Oracle""" + + COMPATIBLE_CONTRACT_VERSION = 1 + COMPATIBLE_CONSENSUS_VERSION = 1 diff --git a/src/modules/oracles/staking_modules/curated/entrypoint.py b/src/modules/oracles/staking_modules/curated/entrypoint.py new file mode 100644 index 000000000..8ba37a353 --- /dev/null +++ b/src/modules/oracles/staking_modules/curated/entrypoint.py @@ -0,0 +1,17 @@ +from src.modules.oracles.common.runtime import OracleWeb3Config, build_oracle_web3, run_oracle_module +from src.modules.oracles.staking_modules.curated.cm import CMPerformanceOracle +from src.runtime import log_startup, start_observability +from src.types import OracleModule + + +def run() -> None: + log_startup(OracleModule.CM) + start_observability() + + web3 = build_oracle_web3(OracleWeb3Config( + use_lido_contracts=False, + use_staking_module_contracts=True, + use_ipfs=True, + use_performance_client=True, + )) + run_oracle_module(CMPerformanceOracle(web3)) diff --git a/src/modules/sidecars/__init__.py b/src/modules/sidecars/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/sidecars/performance/__init__.py b/src/modules/sidecars/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/sidecars/performance/collector/__init__.py b/src/modules/sidecars/performance/collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/performance/collector/checkpoint.py b/src/modules/sidecars/performance/collector/checkpoint.py similarity index 98% rename from src/modules/performance/collector/checkpoint.py rename to src/modules/sidecars/performance/collector/checkpoint.py index 89aaea3b6..be243973c 100644 --- a/src/modules/performance/collector/checkpoint.py +++ b/src/modules/sidecars/performance/collector/checkpoint.py @@ -10,9 +10,9 @@ from src import variables from src.constants import SLOTS_PER_HISTORICAL_ROOT, EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SYNC_COMMITTEE_SIZE -from src.modules.performance.common.types import ProposalDuty, SyncDuty, AttDutyMisses -from src.modules.performance.common.db import DutiesDB -from src.modules.submodules.types import ZERO_HASH +from src.modules.sidecars.performance.common.types import ProposalDuty, SyncDuty, AttDutyMisses +from src.modules.sidecars.performance.common.db import DutiesDB +from src.modules.common.types import ZERO_HASH from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import SyncCommittee, SyncAggregate from src.utils.blockstamp import build_blockstamp @@ -94,7 +94,7 @@ def __iter__(self): class SyncCommitteesCache(UserDict): - max_size = max(2, variables.CSM_ORACLE_MAX_CONCURRENCY) + max_size = max(2, variables.PERFORMANCE_COLLECTOR_MAX_CONCURRENCY) def __setitem__(self, sync_committee_period: int, value: SyncCommittee): if len(self) >= self.max_size: @@ -222,7 +222,7 @@ def _process( unprocessed_epochs: list[EpochNumber], epochs_roots_to_check: dict[EpochNumber, tuple[list[SlotBlockRoot], list[SlotBlockRoot]]], ): - executor = ThreadPoolExecutor(max_workers=variables.CSM_ORACLE_MAX_CONCURRENCY) + executor = ThreadPoolExecutor(max_workers=variables.PERFORMANCE_COLLECTOR_MAX_CONCURRENCY) try: futures = { executor.submit( diff --git a/src/modules/performance/collector/collector.py b/src/modules/sidecars/performance/collector/collector.py similarity index 70% rename from src/modules/performance/collector/collector.py rename to src/modules/sidecars/performance/collector/collector.py index 895a1700f..b56c14314 100644 --- a/src/modules/performance/collector/collector.py +++ b/src/modules/sidecars/performance/collector/collector.py @@ -1,41 +1,70 @@ import logging +import traceback +from contextlib import contextmanager +from typing import Iterator + +from requests.exceptions import ConnectionError as RequestsConnectionError +from timeout_decorator import TimeoutError as DecoratorTimeoutError from src import variables -from src.modules.performance.collector.checkpoint import ( +from src.modules.common.daemon_module import DaemonModule +from src.modules.common.types import ModuleExecuteDelay, ChainConfig +from src.modules.sidecars.performance.collector.checkpoint import ( FrameCheckpointsIterator, FrameCheckpointProcessor, ) -from src.modules.performance.common.db import DutiesDB -from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay -from src.modules.submodules.types import ChainConfig -from src.types import BlockStamp, EpochNumber +from src.modules.sidecars.performance.common.db import DutiesDB +from src.providers.consensus.client import ConsensusClient +from src.providers.http_provider import NotOkResponse +from src.types import BlockStamp, EpochNumber, SlotNumber +from src.utils.slot import InconsistentData, NoSlotsAvailable, SlotNotFinalized from src.utils.web3converter import ChainConverter logger = logging.getLogger(__name__) -class PerformanceCollector(BaseModule): +class PerformanceCollector(DaemonModule): """ Continuously collects performance data from Consensus Layer into db for the given epoch range. """ + _slot_threshold = SlotNumber(0) + # Timestamp of the last epochs demand update - last_epochs_demand_update: int = 0 + last_epochs_demand_update: int | None = None - def __init__(self, w3): - super().__init__(w3) + def __init__(self, cc: ConsensusClient): + self.cc = cc.cc if hasattr(cc, "cc") else cc self.db = DutiesDB( connect_timeout=variables.PERFORMANCE_COLLECTOR_DB_CONNECTION_TIMEOUT, statement_timeout_ms=variables.PERFORMANCE_COLLECTOR_DB_STATEMENT_TIMEOUT_MS, ) self.last_epochs_demand_update = self.get_epochs_demand_max_updated_at() - def refresh_contracts(self): - # No need to refresh contracts for this module. There are no contracts used. - return None + def _get_consensus_client(self): + """Returns consensus client""" + return self.cc + + @contextmanager + def exception_handler(self) -> Iterator[None]: + """Context manager for handling Performance Collector exceptions""" + try: + yield + except DecoratorTimeoutError as error: + logger.error({'msg': 'Performance collector do not respond.', 'error': str(error)}) + except RequestsConnectionError as error: + logger.error({'msg': 'Connection error.', 'error': str(error)}) + except NotOkResponse as error: + logger.error({'msg': ''.join(traceback.format_exception(error))}) + except (NoSlotsAvailable, SlotNotFinalized, InconsistentData) as error: + logger.error({'msg': 'Inconsistent response from consensus layer node.', 'error': str(error)}) + except ValueError as error: + logger.error({'msg': 'Unexpected error.', 'error': str(error)}) + except Exception as error: + raise error def _build_converter(self) -> ChainConverter: - cc_spec = self.w3.cc.get_config_spec() - genesis = self.w3.cc.get_genesis() + cc_spec = self.cc.get_config_spec() + genesis = self.cc.get_genesis() chain_cfg = ChainConfig( slots_per_epoch=cc_spec.SLOTS_PER_EPOCH, seconds_per_slot=cc_spec.SECONDS_PER_SLOT, @@ -63,7 +92,7 @@ def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecute end_epoch, finalized_epoch, ) - processor = FrameCheckpointProcessor(self.w3.cc, self.db, converter, last_finalized_blockstamp) + processor = FrameCheckpointProcessor(self.cc, self.db, converter, last_finalized_blockstamp) checkpoint_count = 0 for checkpoint in checkpoints: @@ -74,7 +103,7 @@ def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecute 'checkpoint_slot': checkpoint.slot, 'processed_epochs': processed_epochs }) - # Reset BaseOracle cycle timeout to avoid timeout errors during long checkpoints processing + # Reset base cycle timeout to avoid timeout errors during long checkpoints processing self._reset_cycle_timeout() if self.new_epochs_range_demand_appeared(): @@ -121,7 +150,7 @@ def define_epochs_to_process_range(self, finalized_epoch: EpochNumber) -> tuple[ self.db.delete_demand(demand.consumer) # There is no sense to lower start_epoch because the demand is already satisfied (data is in the DB) continue - start_epoch = min(start_epoch, demand.l_epoch) + start_epoch = min(start_epoch, EpochNumber(demand.l_epoch)) missing_epochs = self.db.missing_epochs_in(start_epoch, end_epoch) if not missing_epochs: @@ -151,12 +180,12 @@ def define_epochs_to_process_range(self, finalized_epoch: EpochNumber) -> tuple[ def new_epochs_range_demand_appeared(self) -> bool: max_updated_at = self.get_epochs_demand_max_updated_at() - updated = self.last_epochs_demand_update != max_updated_at + updated = max_updated_at is not None and self.last_epochs_demand_update != max_updated_at if updated: self.last_epochs_demand_update = max_updated_at return True return False - def get_epochs_demand_max_updated_at(self) -> int: + def get_epochs_demand_max_updated_at(self) -> int | None: max_updated_at = self.db.get_epochs_demands_max_updated_at() - return int(max_updated_at) if max_updated_at is not None else 0 + return int(max_updated_at) if max_updated_at is not None else None diff --git a/src/modules/sidecars/performance/collector/entrypoint.py b/src/modules/sidecars/performance/collector/entrypoint.py new file mode 100644 index 000000000..c77a3a92f --- /dev/null +++ b/src/modules/sidecars/performance/collector/entrypoint.py @@ -0,0 +1,22 @@ +from src import variables +from src.modules.sidecars.performance.collector.collector import PerformanceCollector +from src.providers.consensus.client import ConsensusClient +from src.runtime import log_startup, start_observability +from src.types import OracleModule + + +def _build_consensus_client() -> ConsensusClient: + return ConsensusClient( + variables.CONSENSUS_CLIENT_URI, + variables.HTTP_REQUEST_TIMEOUT_CONSENSUS, + variables.HTTP_REQUEST_RETRY_COUNT_CONSENSUS, + variables.HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS, + ) + + +def run() -> None: + log_startup(OracleModule.PERFORMANCE_COLLECTOR) + start_observability() + + collector = PerformanceCollector(_build_consensus_client()) + collector.run_as_daemon() diff --git a/src/modules/sidecars/performance/common/__init__.py b/src/modules/sidecars/performance/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/performance/common/db.py b/src/modules/sidecars/performance/common/db.py similarity index 98% rename from src/modules/performance/common/db.py rename to src/modules/sidecars/performance/common/db.py index 0cf32ad15..094ba752e 100644 --- a/src/modules/performance/common/db.py +++ b/src/modules/sidecars/performance/common/db.py @@ -6,7 +6,7 @@ from sqlmodel import SQLModel, Field, Session, create_engine, select, col from src import variables -from src.modules.performance.common.types import ProposalDuty, SyncDuty, AttDutyMisses +from src.modules.sidecars.performance.common.types import ProposalDuty, SyncDuty, AttDutyMisses from src.types import EpochNumber from src.utils.range import sequence diff --git a/src/modules/performance/common/types.py b/src/modules/sidecars/performance/common/types.py similarity index 100% rename from src/modules/performance/common/types.py rename to src/modules/sidecars/performance/common/types.py diff --git a/src/modules/sidecars/performance/web/__init__.py b/src/modules/sidecars/performance/web/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/sidecars/performance/web/entrypoint.py b/src/modules/sidecars/performance/web/entrypoint.py new file mode 100644 index 000000000..c1e523ce7 --- /dev/null +++ b/src/modules/sidecars/performance/web/entrypoint.py @@ -0,0 +1,10 @@ +from src.metrics.logging import logging +from src.modules.sidecars.performance.web.server import serve +from src.variables import PERFORMANCE_WEB_SERVER_API_PORT + +logger = logging.getLogger(__name__) + + +def run() -> int: + logger.info({'msg': f'Starting Performance Web Server on port {PERFORMANCE_WEB_SERVER_API_PORT}'}) + return serve() diff --git a/src/modules/performance/web/metrics.py b/src/modules/sidecars/performance/web/metrics.py similarity index 100% rename from src/modules/performance/web/metrics.py rename to src/modules/sidecars/performance/web/metrics.py diff --git a/src/modules/performance/web/middleware.py b/src/modules/sidecars/performance/web/middleware.py similarity index 100% rename from src/modules/performance/web/middleware.py rename to src/modules/sidecars/performance/web/middleware.py diff --git a/src/modules/performance/web/server.py b/src/modules/sidecars/performance/web/server.py similarity index 94% rename from src/modules/performance/web/server.py rename to src/modules/sidecars/performance/web/server.py index 10c0d21a9..cc65744c1 100644 --- a/src/modules/performance/web/server.py +++ b/src/modules/sidecars/performance/web/server.py @@ -6,8 +6,8 @@ from pydantic import BaseModel import gunicorn.app.base -from src.modules.performance.common.db import DutiesDB, Duty, EpochsDemand -from src.modules.performance.web.middleware import RequestTimeoutMiddleware +from src.modules.sidecars.performance.common.db import DutiesDB, Duty, EpochsDemand +from src.modules.sidecars.performance.web.middleware import RequestTimeoutMiddleware from src.variables import ( PERFORMANCE_WEB_SERVER_API_HOST, PERFORMANCE_WEB_SERVER_API_PORT, @@ -21,7 +21,7 @@ PERFORMANCE_WEB_SERVER_TIMEOUT, PERFORMANCE_WEB_SERVER_KEEPALIVE, ) -from src.modules.performance.web.metrics import attach_metrics +from src.modules.sidecars.performance.web.metrics import attach_metrics from src.types import EpochNumber logger = logging.getLogger(__name__) @@ -82,8 +82,8 @@ def validate_range_size(l_epoch: EpochNumber, r_epoch: EpochNumber) -> None: def query_epoch_range( - from_epoch: EpochNumber = Query(..., alias="from"), - to_epoch: EpochNumber = Query(..., alias="to"), + from_epoch: EpochNumber = Query(..., alias="from"), + to_epoch: EpochNumber = Query(..., alias="to"), ) -> tuple[EpochNumber, EpochNumber]: validate_epoch_bounds(from_epoch, to_epoch) return from_epoch, to_epoch diff --git a/src/providers/consensus/client.py b/src/providers/consensus/client.py index 37539d33c..09ef3074f 100644 --- a/src/providers/consensus/client.py +++ b/src/providers/consensus/client.py @@ -113,7 +113,7 @@ def get_block_details(self, state_id: SlotNumber | BlockRoot) -> BlockDetailsRes ) return BlockDetailsResponse.from_response(**data) - @lru_cache(maxsize=variables.CSM_ORACLE_MAX_CONCURRENCY * 32 * 2) # threads count * blocks * epochs to check duties + @lru_cache(maxsize=variables.PERFORMANCE_COLLECTOR_MAX_CONCURRENCY * 32 * 2) # threads count * blocks * epochs to check duties def get_block_attestations_and_sync( self, state_id: SlotNumber | BlockRoot ) -> tuple[list[BlockAttestation], SyncAggregate]: diff --git a/src/providers/execution/contracts/accounting.py b/src/providers/execution/contracts/accounting.py index c9905ea9d..f62b04f53 100644 --- a/src/providers/execution/contracts/accounting.py +++ b/src/providers/execution/contracts/accounting.py @@ -2,7 +2,7 @@ from web3.types import BlockIdentifier -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.types import ( ReportSimulationPayload, ReportSimulationResults, ) diff --git a/src/providers/execution/contracts/accounting_oracle.py b/src/providers/execution/contracts/accounting_oracle.py index 6e047a6ea..3feaff3c1 100644 --- a/src/providers/execution/contracts/accounting_oracle.py +++ b/src/providers/execution/contracts/accounting_oracle.py @@ -4,7 +4,7 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.modules.accounting.types import AccountingProcessingState +from src.modules.oracles.accounting.types import AccountingProcessingState from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/execution/contracts/burner.py b/src/providers/execution/contracts/burner.py index bd78b7e15..cb5fa99bc 100644 --- a/src/providers/execution/contracts/burner.py +++ b/src/providers/execution/contracts/burner.py @@ -3,7 +3,7 @@ from web3.types import BlockIdentifier -from src.modules.accounting.types import SharesRequestedToBurn +from src.modules.oracles.accounting.types import SharesRequestedToBurn from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/execution/contracts/cs_parameters_registry.py b/src/providers/execution/contracts/cs_parameters_registry.py index 1dc7b0631..6c29dc65b 100644 --- a/src/providers/execution/contracts/cs_parameters_registry.py +++ b/src/providers/execution/contracts/cs_parameters_registry.py @@ -5,7 +5,7 @@ from web3.types import BlockIdentifier from src.constants import TOTAL_BASIS_POINTS, ATTESTATIONS_WEIGHT, BLOCKS_WEIGHT, SYNC_WEIGHT -from src.modules.csm.state import ValidatorDuties +from src.modules.oracles.staking_modules.common.state import ValidatorDuties from src.providers.execution.base_interface import ContractInterface from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/exit_bus_oracle.py b/src/providers/execution/contracts/exit_bus_oracle.py index a4c224ec1..77e820507 100644 --- a/src/providers/execution/contracts/exit_bus_oracle.py +++ b/src/providers/execution/contracts/exit_bus_oracle.py @@ -2,7 +2,7 @@ from web3.types import BlockIdentifier -from src.modules.ejector.types import EjectorProcessingState +from src.modules.oracles.ejector.types import EjectorProcessingState from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/hash_consensus.py b/src/providers/execution/contracts/hash_consensus.py index 0d909f549..bac16744d 100644 --- a/src/providers/execution/contracts/hash_consensus.py +++ b/src/providers/execution/contracts/hash_consensus.py @@ -6,7 +6,7 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig +from src.modules.common.types import ChainConfig, CurrentFrame, FrameConfig from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/execution/contracts/lazy_oracle.py b/src/providers/execution/contracts/lazy_oracle.py index 6e4f7b0ae..6074e49be 100644 --- a/src/providers/execution/contracts/lazy_oracle.py +++ b/src/providers/execution/contracts/lazy_oracle.py @@ -5,7 +5,7 @@ from web3.types import BlockIdentifier from src import variables -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.types import ( OnChainIpfsVaultReportData, ValidatorStage, VaultInfo, diff --git a/src/providers/execution/contracts/lido.py b/src/providers/execution/contracts/lido.py index bd5c35a74..34710b8ae 100644 --- a/src/providers/execution/contracts/lido.py +++ b/src/providers/execution/contracts/lido.py @@ -2,7 +2,7 @@ from web3.types import Wei, BlockIdentifier -from src.modules.accounting.types import BeaconStat +from src.modules.oracles.accounting.types import BeaconStat from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/oracle_report_sanity_checker.py b/src/providers/execution/contracts/oracle_report_sanity_checker.py index 499676ca2..fa1510bf4 100644 --- a/src/providers/execution/contracts/oracle_report_sanity_checker.py +++ b/src/providers/execution/contracts/oracle_report_sanity_checker.py @@ -3,7 +3,7 @@ from web3.types import BlockIdentifier -from src.modules.accounting.types import OracleReportLimits +from src.modules.oracles.accounting.types import OracleReportLimits from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/execution/contracts/staking_router.py b/src/providers/execution/contracts/staking_router.py index 628250235..8dda00bf9 100644 --- a/src/providers/execution/contracts/staking_router.py +++ b/src/providers/execution/contracts/staking_router.py @@ -1,6 +1,6 @@ import logging -from src.modules.accounting.types import StakingFeeAggregateDistribution +from src.modules.oracles.accounting.types import StakingFeeAggregateDistribution from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/vault_hub.py b/src/providers/execution/contracts/vault_hub.py index c54cbcd23..82e5862fe 100644 --- a/src/providers/execution/contracts/vault_hub.py +++ b/src/providers/execution/contracts/vault_hub.py @@ -2,7 +2,7 @@ from eth_typing import BlockNumber -from src.modules.accounting.events import ( +from src.modules.oracles.accounting.events import ( BadDebtSocializedEvent, BadDebtWrittenOffToBeInternalizedEvent, BurnedSharesOnVaultEvent, diff --git a/src/providers/execution/contracts/withdrawal_queue_nft.py b/src/providers/execution/contracts/withdrawal_queue_nft.py index dcf3931cd..bd9be5380 100644 --- a/src/providers/execution/contracts/withdrawal_queue_nft.py +++ b/src/providers/execution/contracts/withdrawal_queue_nft.py @@ -3,7 +3,7 @@ from web3.types import Wei, BlockIdentifier -from src.modules.accounting.types import BatchState, WithdrawalRequestStatus +from src.modules.oracles.accounting.types import BatchState, WithdrawalRequestStatus from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/performance/client.py b/src/providers/performance/client.py index d49740363..454a44ca5 100644 --- a/src/providers/performance/client.py +++ b/src/providers/performance/client.py @@ -1,5 +1,5 @@ from src.metrics.prometheus.basic import PERFORMANCE_REQUESTS_DURATION -from src.modules.performance.common.db import Duty, EpochsDemand +from src.modules.sidecars.performance.common.db import Duty, EpochsDemand from src.providers.http_provider import ( HTTPProvider, NotOkResponse, diff --git a/src/runtime.py b/src/runtime.py new file mode 100644 index 000000000..9a3019290 --- /dev/null +++ b/src/runtime.py @@ -0,0 +1,31 @@ +from prometheus_client import start_http_server + +from src import variables +from src.metrics.healthcheck_server import start_pulse_server +from src.metrics.logging import logging +from src.metrics.prometheus.basic import BUILD_INFO, ENV_VARIABLES_INFO +from src.utils.build import get_build_info + +logger = logging.getLogger(__name__) + + +def log_startup(module_name: str) -> None: + build_info = get_build_info() + logger.info({ + 'msg': 'Oracle startup.', + 'variables': { + **build_info, + 'module': module_name, + **variables.PUBLIC_ENV_VARS, + }, + }) + ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS) + BUILD_INFO.info(build_info) + + +def start_observability() -> None: + logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) + start_pulse_server() + + logger.info({'msg': f'Start http server with prometheus metrics on port {variables.PROMETHEUS_PORT}'}) + start_http_server(variables.PROMETHEUS_PORT) diff --git a/src/services/bunker.py b/src/services/bunker.py index 8facb73ab..3f1c2e63e 100644 --- a/src/services/bunker.py +++ b/src/services/bunker.py @@ -10,8 +10,8 @@ ALL_SLASHED_VALIDATORS, LIDO_SLASHED_VALIDATORS, ) -from src.modules.accounting.types import ReportSimulationResults -from src.modules.submodules.consensus import FrameConfig, ChainConfig +from src.modules.oracles.accounting.types import ReportSimulationResults +from src.modules.oracles.common.consensus import FrameConfig, ChainConfig from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty from src.services.bunker_cases.types import BunkerConfig diff --git a/src/services/bunker_cases/abnormal_cl_rebase.py b/src/services/bunker_cases/abnormal_cl_rebase.py index 72941e5a7..302a81235 100644 --- a/src/services/bunker_cases/abnormal_cl_rebase.py +++ b/src/services/bunker_cases/abnormal_cl_rebase.py @@ -6,7 +6,7 @@ from web3.types import EventData from src.constants import EFFECTIVE_BALANCE_INCREMENT, LIDO_DEPOSIT_AMOUNT -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey from src.services.bunker_cases.types import BunkerConfig diff --git a/src/services/exit_order_iterator.py b/src/services/exit_order_iterator.py index 64350bac0..17666d767 100644 --- a/src/services/exit_order_iterator.py +++ b/src/services/exit_order_iterator.py @@ -3,7 +3,7 @@ from src.constants import TOTAL_BASIS_POINTS from src.metrics.prometheus.duration_meter import duration_meter -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.services.validator_state import LidoValidatorStateService from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId from src.utils.validator_state import is_on_exit diff --git a/src/services/prediction.py b/src/services/prediction.py index 8896b5f98..854f0a822 100644 --- a/src/services/prediction.py +++ b/src/services/prediction.py @@ -2,7 +2,7 @@ from web3.types import EventData, Wei -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.execution.exceptions import InconsistentEvents from src.types import ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/services/safe_border.py b/src/services/safe_border.py index 47ed5fcc9..eda9d4a7d 100644 --- a/src/services/safe_border.py +++ b/src/services/safe_border.py @@ -3,7 +3,7 @@ from src.constants import EPOCHS_PER_SLASHINGS_VECTOR, MIN_VALIDATOR_WITHDRAWABILITY_DELAY from src.metrics.prometheus.duration_meter import duration_meter -from src.modules.submodules.consensus import ChainConfig, FrameConfig +from src.modules.oracles.common.consensus import ChainConfig, FrameConfig from src.types import EpochNumber, FrameNumber, ReferenceBlockStamp, SlotNumber from src.utils.slot import get_blockstamp from src.utils.web3converter import Web3Converter diff --git a/src/services/staking_vaults.py b/src/services/staking_vaults.py index bcb942829..227640c58 100644 --- a/src/services/staking_vaults.py +++ b/src/services/staking_vaults.py @@ -13,7 +13,7 @@ MIN_DEPOSIT_AMOUNT, TOTAL_BASIS_POINTS, ) -from src.modules.accounting.events import ( +from src.modules.oracles.accounting.events import ( BadDebtSocializedEvent, BadDebtWrittenOffToBeInternalizedEvent, BurnedSharesOnVaultEvent, @@ -24,7 +24,7 @@ VaultRebalancedEvent, sort_events, ) -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.types import ( BLOCKS_PER_YEAR, ExtraValue, MerkleValue, @@ -41,7 +41,7 @@ VaultToValidators, VaultTreeNode, ) -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.common.types import ChainConfig, FrameConfig from src.providers.consensus.types import PendingDeposit, Validator from src.providers.ipfs import CID from src.types import FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber diff --git a/src/services/validator_state.py b/src/services/validator_state.py index 0881eb179..c05167263 100644 --- a/src/services/validator_state.py +++ b/src/services/validator_state.py @@ -3,7 +3,7 @@ from functools import reduce from src.metrics.prometheus.accounting import ACCOUNTING_EXITED_VALIDATORS -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.types import OperatorsValidatorCount, ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.events import get_events_in_past diff --git a/src/services/withdrawal.py b/src/services/withdrawal.py index ccc799600..63501aae8 100644 --- a/src/services/withdrawal.py +++ b/src/services/withdrawal.py @@ -5,8 +5,8 @@ from src.web3py.types import Web3 from src.types import ReferenceBlockStamp, FinalizationBatches from src.services.safe_border import SafeBorder -from src.modules.submodules.consensus import ChainConfig, FrameConfig -from src.modules.accounting.types import BatchState +from src.modules.oracles.common.consensus import ChainConfig, FrameConfig +from src.modules.oracles.accounting.types import BatchState class Withdrawal: diff --git a/src/types.py b/src/types.py index dfb2e8f74..cd19bc7f9 100644 --- a/src/types.py +++ b/src/types.py @@ -11,6 +11,7 @@ class OracleModule(StrEnum): EJECTOR = 'ejector' CHECK = 'check' CSM = 'csm' + CM = 'cm' PERFORMANCE_WEB_SERVER = 'performance_web_server' PERFORMANCE_COLLECTOR = 'performance_collector' diff --git a/src/utils/apr.py b/src/utils/apr.py index 617552417..06fb4cad8 100644 --- a/src/utils/apr.py +++ b/src/utils/apr.py @@ -1,7 +1,7 @@ from decimal import Decimal from src.constants import SHARE_RATE_PRECISION_E27 -from src.modules.accounting.types import SECONDS_IN_YEAR, Shares +from src.modules.oracles.accounting.types import SECONDS_IN_YEAR, Shares def calculate_gross_core_apr( diff --git a/src/utils/web3converter.py b/src/utils/web3converter.py index 58a705e97..555c931c6 100644 --- a/src/utils/web3converter.py +++ b/src/utils/web3converter.py @@ -1,5 +1,6 @@ from src.types import SlotNumber, EpochNumber, FrameNumber -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.common.types import ChainConfig +from src.modules.common.types import FrameConfig def epoch_from_slot(slot: SlotNumber, slots_per_epoch: int) -> EpochNumber: diff --git a/src/variables.py b/src/variables.py index c8c3b3c7d..5bc9ec537 100644 --- a/src/variables.py +++ b/src/variables.py @@ -36,10 +36,10 @@ # - App specific - LIDO_LOCATOR_ADDRESS: Final = os.getenv('LIDO_LOCATOR_ADDRESS') -CSM_MODULE_ADDRESS: Final = os.getenv('CSM_MODULE_ADDRESS') +CS_MODULE_ADDRESS: Final = os.getenv('CS_MODULE_ADDRESS') +CURATED_MODULE_ADDRESS: Final = os.getenv('CURATED_MODULE_ADDRESS') FINALIZATION_BATCH_MAX_REQUEST_COUNT: Final = int(os.getenv('FINALIZATION_BATCH_MAX_REQUEST_COUNT', 1000)) EL_REQUESTS_BATCH_SIZE: Final = int(os.getenv('EL_REQUESTS_BATCH_SIZE', 500)) -CSM_ORACLE_MAX_CONCURRENCY: Final = min(32, int(os.getenv('CSM_ORACLE_MAX_CONCURRENCY', 2))) # We add some gas to the transaction to be sure that we have enough gas to execute corner cases # eg when we tried to submit a few reports in a single block @@ -122,6 +122,7 @@ PERFORMANCE_WEB_SERVER_TIMEOUT: Final = int(os.getenv('PERFORMANCE_WEB_SERVER_TIMEOUT', 30)) PERFORMANCE_WEB_SERVER_KEEPALIVE: Final = int(os.getenv('PERFORMANCE_WEB_SERVER_KEEPALIVE', 2)) +PERFORMANCE_COLLECTOR_MAX_CONCURRENCY: Final = min(32, int(os.getenv('PERFORMANCE_COLLECTOR_MAX_CONCURRENCY', 2))) PERFORMANCE_COLLECTOR_DB_RETENTION_EPOCHS: Final = int(os.getenv('PERFORMANCE_COLLECTOR_DB_RETENTION_EPOCHS', 28 * 225 * 6)) PERFORMANCE_COLLECTOR_DB_CONNECTION_TIMEOUT: Final = int(os.getenv('PERFORMANCE_COLLECTOR_DB_CONNECTION_TIMEOUT', 30)) PERFORMANCE_COLLECTOR_DB_STATEMENT_TIMEOUT_MS: Final = int(os.getenv('PERFORMANCE_COLLECTOR_DB_STATEMENT_TIMEOUT_MS', 10_000)) @@ -142,11 +143,14 @@ def check_all_required_variables(module: OracleModule): errors = check_uri_required_variables() - if not LIDO_LOCATOR_ADDRESS: + if module is not OracleModule.CSM and module is not OracleModule.CM and not LIDO_LOCATOR_ADDRESS: errors.append('LIDO_LOCATOR_ADDRESS') - if module is OracleModule.CSM and not CSM_MODULE_ADDRESS: - errors.append('CSM_MODULE_ADDRESS') + if module is OracleModule.CSM and not CS_MODULE_ADDRESS: + errors.append('CS_MODULE_ADDRESS') + + if module is OracleModule.CM and not CURATED_MODULE_ADDRESS: + errors.append('CURATED_MODULE_ADDRESS') return errors @@ -200,10 +204,10 @@ def raise_from_errors(errors): for key, value in { 'ACCOUNT': 'Dry' if ACCOUNT is None else ACCOUNT.address, 'LIDO_LOCATOR_ADDRESS': LIDO_LOCATOR_ADDRESS, - 'CSM_MODULE_ADDRESS': CSM_MODULE_ADDRESS, + 'CS_MODULE_ADDRESS': CS_MODULE_ADDRESS, + 'CURATED_MODULE_ADDRESS': CURATED_MODULE_ADDRESS, 'FINALIZATION_BATCH_MAX_REQUEST_COUNT': FINALIZATION_BATCH_MAX_REQUEST_COUNT, 'EL_REQUESTS_BATCH_SIZE': EL_REQUESTS_BATCH_SIZE, - 'CSM_ORACLE_MAX_CONCURRENCY': CSM_ORACLE_MAX_CONCURRENCY, 'TX_GAS_ADDITION': TX_GAS_ADDITION, 'EVENTS_SEARCH_STEP': EVENTS_SEARCH_STEP, 'MIN_PRIORITY_FEE': MIN_PRIORITY_FEE, @@ -238,6 +242,7 @@ def raise_from_errors(errors): 'PERFORMANCE_WEB_SERVER_MAX_REQUESTS': PERFORMANCE_WEB_SERVER_MAX_REQUESTS, 'PERFORMANCE_WEB_SERVER_TIMEOUT': PERFORMANCE_WEB_SERVER_TIMEOUT, 'PERFORMANCE_WEB_SERVER_KEEPALIVE': PERFORMANCE_WEB_SERVER_KEEPALIVE, + 'PERFORMANCE_COLLECTOR_MAX_CONCURRENCY': PERFORMANCE_COLLECTOR_MAX_CONCURRENCY, 'PERFORMANCE_COLLECTOR_DB_RETENTION_EPOCHS': PERFORMANCE_COLLECTOR_DB_RETENTION_EPOCHS, 'PERFORMANCE_COLLECTOR_DB_CONNECTION_TIMEOUT': PERFORMANCE_COLLECTOR_DB_CONNECTION_TIMEOUT, 'PERFORMANCE_DB_HOST': PERFORMANCE_DB_HOST, diff --git a/src/web3py/extensions/__init__.py b/src/web3py/extensions/__init__.py index 01ce1d1a6..4f20b2230 100644 --- a/src/web3py/extensions/__init__.py +++ b/src/web3py/extensions/__init__.py @@ -4,6 +4,6 @@ from src.web3py.extensions.contracts import LidoContracts from src.web3py.extensions.lido_validators import LidoValidatorsProvider from src.web3py.extensions.fallback import FallbackProviderModule -from src.web3py.extensions.csm import CSM, LazyCSM +from src.web3py.extensions.staking_module import StakingModuleContracts from src.web3py.extensions.ipfs import IPFS from src.web3py.extensions.performance import PerformanceClientModule diff --git a/src/web3py/extensions/csm.py b/src/web3py/extensions/staking_module.py similarity index 74% rename from src/web3py/extensions/csm.py rename to src/web3py/extensions/staking_module.py index 4262e2c03..3a09ecb6e 100644 --- a/src/web3py/extensions/csm.py +++ b/src/web3py/extensions/staking_module.py @@ -1,5 +1,4 @@ import logging -from functools import partial from time import sleep from typing import cast @@ -17,13 +16,20 @@ from src.providers.execution.contracts.cs_parameters_registry import CSParametersRegistryContract, CurveParams from src.providers.execution.contracts.cs_strikes import CSStrikesContract from src.providers.ipfs import CID, CIDv0, CIDv1, is_cid_v0 -from src.utils.lazy_object_proxy import LazyObjectProxy from src.types import BlockStamp, NodeOperatorId, SlotNumber logger = logging.getLogger(__name__) -class CSM(Module): +class StakingModuleContracts(Module): + """ + Web3 extension for staking module contracts interaction. + + Automatically determines which module to use based on environment variables: + - If CS_MODULE_ADDRESS is set: uses CS module + - If CURATED_MODULE_ADDRESS is set: uses Curated module + - Error if both or neither are set + """ w3: Web3 oracle: CSFeeOracleContract @@ -38,11 +44,25 @@ class CSM(Module): def __init__(self, w3: Web3) -> None: super().__init__(w3) + cs_address_set = bool(variables.CS_MODULE_ADDRESS) + curated_address_set = bool(variables.CURATED_MODULE_ADDRESS) + + if cs_address_set and curated_address_set: + raise ValueError("Both CS_MODULE_ADDRESS and CURATED_MODULE_ADDRESS are set. Only one should be provided.") + + if cs_address_set: + self._module_address = variables.CS_MODULE_ADDRESS + elif curated_address_set: + self._module_address = variables.CURATED_MODULE_ADDRESS + else: + raise ValueError("Neither CS_MODULE_ADDRESS nor CURATED_MODULE_ADDRESS is set") + + self._contract_addresses: tuple[str, ...] | None = None self._load_contracts() - def get_csm_last_processing_ref_slot(self, blockstamp: BlockStamp) -> SlotNumber: + def get_last_processing_ref_slot(self, blockstamp: BlockStamp) -> SlotNumber: result = self.oracle.get_last_processing_ref_slot(blockstamp.block_hash) - FRAME_PREV_REPORT_REF_SLOT.labels("csm_oracle").set(result) + FRAME_PREV_REPORT_REF_SLOT.labels("staking_module").set(result) return result def get_rewards_tree_root(self, blockstamp: BlockStamp) -> HexBytes: @@ -79,7 +99,7 @@ def _load_contracts(self) -> None: self.module = cast( CSModuleContract, self.w3.eth.contract( - address=variables.CSM_MODULE_ADDRESS, # type: ignore + address=self._module_address, # type: ignore ContractFactoryClass=CSModuleContract, decode_tuples=True, ), @@ -129,6 +149,7 @@ def _load_contracts(self) -> None: decode_tuples=True, ), ) + self._contract_addresses = self._get_contract_addresses() return except Web3Exception as e: last_error = e @@ -143,9 +164,22 @@ def _load_contracts(self) -> None: f"after {self.CONTRACT_LOAD_MAX_RETRIES} attempts" ) from last_error - -class LazyCSM(CSM): - """A wrapper around CSM module to achieve lazy-loading behaviour""" - - def __new__(cls, w3: Web3) -> 'LazyCSM': - return LazyObjectProxy(partial(CSM, w3)) # type: ignore + def _get_contract_addresses(self) -> tuple[str, ...]: + return ( + self.module.address, + self.module.accounting(), + self.module.parameters_registry(), + self.accounting.fee_distributor(), + self.fee_distributor.oracle(), + self.oracle.strikes(), + ) + + def has_contract_address_changed(self) -> bool: + current = self._get_contract_addresses() + if self._contract_addresses != current: + self._contract_addresses = current + return True + return False + + def reload_contracts(self) -> None: + self._load_contracts() diff --git a/src/web3py/types.py b/src/web3py/types.py index 713ac6207..ebb712874 100644 --- a/src/web3py/types.py +++ b/src/web3py/types.py @@ -2,7 +2,7 @@ from src.providers.performance.client import PerformanceClient from src.web3py.extensions import ( - CSM, + StakingModuleContracts, ConsensusClientModule, KeysAPIClientModule, LidoContracts, @@ -18,6 +18,6 @@ class Web3(_Web3): transaction: TransactionUtils cc: ConsensusClientModule kac: KeysAPIClientModule - csm: CSM + staking_module: StakingModuleContracts ipfs: IPFS performance: PerformanceClient diff --git a/tests/conftest.py b/tests/conftest.py index c106bb961..c9a4c1b61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,10 @@ from web3 import EthereumTesterProvider from src import variables -from src.main import ipfs_providers +from src.modules.oracles.common.runtime import ipfs_providers from src.providers.execution.base_interface import ContractInterface from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import ( - CSM, ConsensusClientModule, IPFS, KeysAPIClientModule, @@ -81,7 +80,10 @@ def configure_mainnet_tests(request, monkeypatch): ) monkeypatch.setattr(variables, 'LIDO_LOCATOR_ADDRESS', '0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb') - monkeypatch.setattr(variables, 'CSM_MODULE_ADDRESS', '0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F') + monkeypatch.setattr(variables, 'CS_MODULE_ADDRESS', '0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F') + monkeypatch.setattr( + variables, 'CURATED_MODULE_ADDRESS', '0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F' + ) # TODO: replace with actual address yield @@ -104,7 +106,10 @@ def configure_testnet_tests(request, monkeypatch): # TODO: temporary from https://docs.lido.fi/deployed-contracts/hoodi-lidov3, need to # TODO: revert to https://docs.lido.fi/deployed-contracts/hoodi after vaults deploy monkeypatch.setattr(variables, 'LIDO_LOCATOR_ADDRESS', '0x861051869BE0240988918641A9417B10bf4Eed6a') - monkeypatch.setattr(variables, 'CSM_MODULE_ADDRESS', '0x79cef36d84743222f37765204bec41e92a93e59d') + monkeypatch.setattr(variables, 'CS_MODULE_ADDRESS', '0x79cef36d84743222f37765204bec41e92a93e59d') + monkeypatch.setattr( + variables, 'CURATED_MODULE_ADDRESS', '0x79cef36d84743222f37765204bec41e92a93e59d' + ) # TODO: replace with actual address yield @@ -117,7 +122,8 @@ def web3(monkeypatch) -> Generator[Web3, None, None]: tweak_w3_contracts(w3) monkeypatch.setattr(variables, 'LIDO_LOCATOR_ADDRESS', DUMMY_ADDRESS) - monkeypatch.setattr(variables, 'CSM_MODULE_ADDRESS', DUMMY_ADDRESS) + monkeypatch.setattr(variables, 'CS_MODULE_ADDRESS', DUMMY_ADDRESS) + monkeypatch.setattr(variables, 'CURATED_MODULE_ADDRESS', DUMMY_ADDRESS) def create_contract_mock(*args, **kwargs): """ @@ -145,7 +151,6 @@ def create_contract_mock(*args, **kwargs): # Mocked on the contract level, see create_contract_mock 'lido_contracts': LidoContracts, 'transaction': TransactionUtils, - 'csm': CSM, 'lido_validators': LidoValidatorsProvider, # Modules relying on network level highly - mocked fully 'cc': lambda: Mock(spec=ConsensusClientModule), diff --git a/tests/e2e/test_accounting.py b/tests/e2e/test_accounting.py index 8816a391e..ae59e2cda 100644 --- a/tests/e2e/test_accounting.py +++ b/tests/e2e/test_accounting.py @@ -1,6 +1,6 @@ import pytest -from src.modules.accounting.accounting import Accounting +from src.modules.oracles.accounting.accounting import Accounting from tests.e2e.conftest import wait_for_message_appeared diff --git a/tests/factory/base_oracle.py b/tests/factory/base_oracle.py index a3bad2542..d60688747 100644 --- a/tests/factory/base_oracle.py +++ b/tests/factory/base_oracle.py @@ -1,5 +1,5 @@ -from src.modules.accounting.types import AccountingProcessingState -from src.modules.ejector.types import EjectorProcessingState +from src.modules.oracles.accounting.types import AccountingProcessingState +from src.modules.oracles.ejector.types import EjectorProcessingState from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/factory/configs.py b/tests/factory/configs.py index 5dabffd2c..1ce39ac3a 100644 --- a/tests/factory/configs.py +++ b/tests/factory/configs.py @@ -1,5 +1,5 @@ -from src.modules.accounting.types import OracleReportLimits -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.oracles.accounting.types import OracleReportLimits +from src.modules.common.types import ChainConfig, FrameConfig from src.providers.consensus.types import ( AttestationData, BeaconSpecResponse, diff --git a/tests/factory/contract_responses.py b/tests/factory/contract_responses.py index 31a0e2c98..8dee9e4e8 100644 --- a/tests/factory/contract_responses.py +++ b/tests/factory/contract_responses.py @@ -1,4 +1,4 @@ -from src.modules.accounting.types import ReportSimulationResults +from src.modules.oracles.accounting.types import ReportSimulationResults from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/factory/member_info.py b/tests/factory/member_info.py index 89db2b944..047bda6ff 100644 --- a/tests/factory/member_info.py +++ b/tests/factory/member_info.py @@ -1,4 +1,4 @@ -from src.modules.submodules.types import MemberInfo +from src.modules.common.types import MemberInfo from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/fork/conftest.py b/tests/fork/conftest.py index b66914e30..2e405123c 100644 --- a/tests/fork/conftest.py +++ b/tests/fork/conftest.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from pathlib import Path from typing import cast, get_args +from unittest.mock import patch import pytest import xdist @@ -16,10 +17,10 @@ from web3_multi_provider import MultiProvider from src import variables -from src.main import ipfs_providers -from src.modules.submodules.consensus import ConsensusModule -from src.modules.submodules.oracle_module import BaseModule -from src.modules.submodules.types import FrameConfig +from src.modules.oracles.common.consensus import ConsensusModule +from src.modules.oracles.common.oracle_module import BaseModule +from src.modules.common.types import FrameConfig +from src.modules.oracles.common.runtime import ipfs_providers from src.providers.consensus.client import ConsensusClient, LiteralState from src.providers.consensus.types import BlockDetailsResponse, BlockRootResponse from src.providers.execution.contracts.base_oracle import BaseOracleContract @@ -38,13 +39,13 @@ from src.web3py.extensions import ( IPFS, KeysAPIClientModule, - LazyCSM, LidoContracts, LidoValidatorsProvider, TransactionUtils, PerformanceClientModule, FallbackProviderModule, ) +from src.web3py.extensions.staking_module import StakingModuleContracts logger = logging.getLogger('fork_tests') @@ -106,14 +107,14 @@ def set_delay_and_sleep(monkeypatch): @pytest.fixture(autouse=True) def patch_csm_contract_load(monkeypatch): monkeypatch.setattr( - "src.web3py.extensions.CSM.CONTRACT_LOAD_MAX_RETRIES", + "src.web3py.extensions.StakingModuleContracts.CONTRACT_LOAD_MAX_RETRIES", 3, ) monkeypatch.setattr( - "src.web3py.extensions.CSM.CONTRACT_LOAD_RETRY_DELAY", + "src.web3py.extensions.StakingModuleContracts.CONTRACT_LOAD_RETRY_DELAY", 0, ) - logger.info("TESTRUN Patched CSM CONTRACT_LOAD_MAX_RETRIES to 3 and CONTRACT_LOAD_RETRY_DELAY to 0") + logger.info("TESTRUN Patched Staking Module CONTRACT_LOAD_MAX_RETRIES to 3 and CONTRACT_LOAD_RETRY_DELAY to 0") yield @@ -292,7 +293,6 @@ def web3(forked_el_client, patched_cl_client, mocked_ipfs_client): 'lido_contracts': LidoContracts, 'lido_validators': LidoValidatorsProvider, 'transaction': TransactionUtils, - "csm": LazyCSM, # type: ignore[dict-item] 'cc': lambda: patched_cl_client, # type: ignore[dict-item] 'kac': lambda: kac, # type: ignore[dict-item] "ipfs": lambda: mocked_ipfs_client, @@ -302,6 +302,20 @@ def web3(forked_el_client, patched_cl_client, mocked_ipfs_client): yield forked_el_client +@pytest.fixture() +def web3_cs_module(web3): + with patch.object(variables, "CURATED_MODULE_ADDRESS", None): + web3.attach_modules({'staking_module': StakingModuleContracts}) + yield web3 + + +@pytest.fixture() +def web3_curated_module(web3): + with patch.object(variables, "CS_MODULE_ADDRESS", None): + web3.attach_modules({'staking_module': StakingModuleContracts}) + yield web3 + + @pytest.fixture() def mocked_ipfs_client(monkeypatch, forked_el_client): def _publish(self, content: bytes, name: str | None = None) -> CID: diff --git a/tests/fork/test_lido_oracle_cycle.py b/tests/fork/test_lido_oracle_cycle.py index 30fad386e..139ddd396 100644 --- a/tests/fork/test_lido_oracle_cycle.py +++ b/tests/fork/test_lido_oracle_cycle.py @@ -1,8 +1,8 @@ import pytest -from src.modules.accounting.accounting import Accounting -from src.modules.ejector.ejector import Ejector -from src.modules.submodules.types import FrameConfig +from src.modules.oracles.accounting.accounting import Accounting +from src.modules.oracles.ejector.ejector import Ejector +from src.modules.common.types import FrameConfig from src.utils.range import sequence from tests.fork.conftest import first_slot_of_epoch diff --git a/tests/fork/test_csm_oracle_cycle.py b/tests/fork/test_staking_module_oracle_cycle.py similarity index 60% rename from tests/fork/test_csm_oracle_cycle.py rename to tests/fork/test_staking_module_oracle_cycle.py index 680b2d0b8..2cbcd77ee 100644 --- a/tests/fork/test_csm_oracle_cycle.py +++ b/tests/fork/test_staking_module_oracle_cycle.py @@ -1,21 +1,25 @@ from threading import Thread import pytest +import uvicorn from unittest.mock import patch from pathlib import Path from sqlmodel import create_engine from sqlalchemy import JSON -from src.modules.csm.csm import CSOracle -from src.modules.performance.collector.collector import PerformanceCollector -from src.modules.performance.common.db import Duty -from src.modules.performance.web.server import serve -from src.modules.submodules.types import FrameConfig +from src.modules.oracles.staking_modules.community_staking.csm import CSPerformanceOracle +from src.modules.oracles.staking_modules.curated.cm import CMPerformanceOracle +from src.modules.sidecars.performance.collector.collector import PerformanceCollector +from src.modules.common.types import FrameConfig +from src.modules.sidecars.performance.common.db import Duty +from src.modules.sidecars.performance.web.server import app from src.utils.range import sequence from src.web3py.types import Web3 from tests.fork.conftest import first_slot_of_epoch +# pylint: disable=protected-access + @pytest.fixture() def hash_consensus_bin(): @@ -24,8 +28,13 @@ def hash_consensus_bin(): @pytest.fixture() -def csm_module(web3: Web3): - yield CSOracle(web3) +def csm_module(web3_cs_module: Web3): + yield CSPerformanceOracle(web3_cs_module) + + +@pytest.fixture() +def cm_module(web3_curated_module: Web3): + yield CMPerformanceOracle(web3_curated_module) @pytest.fixture() @@ -33,12 +42,19 @@ def performance_local_db(testrun_path): def mock_get_database_url(self): db_path = Path(testrun_path) / "test_duties.db" + db_path.parent.mkdir(parents=True, exist_ok=True) return f"sqlite:///{db_path}" - def mock_init(self): - # pylint: disable=protected-access - self.engine = create_engine(self._get_database_url(), echo=False) - # pylint: disable=protected-access + def mock_build_engine(self, connect_timeout): + return create_engine( + self._get_database_url(), + echo=False, + pool_pre_ping=True, + ) + + def mock_init(self, *args, **kwargs): + self._statement_timeout_ms = kwargs.get('statement_timeout_ms') + self.engine = self._build_engine(kwargs.get('connect_timeout')) self._setup_database() table = Duty.__table__ @@ -46,19 +62,22 @@ def mock_init(self): if col_name in table.c: table.c[col_name].type = JSON() - with patch('src.modules.performance.common.db.DutiesDB._get_database_url', mock_get_database_url): - with patch('src.modules.performance.common.db.DutiesDB.__init__', mock_init): - yield + with patch('src.modules.sidecars.performance.common.db.DutiesDB._get_database_url', mock_get_database_url): + with patch('src.modules.sidecars.performance.common.db.DutiesDB._build_engine', mock_build_engine): + with patch('src.modules.sidecars.performance.common.db.DutiesDB.__init__', mock_init): + yield mock_get_database_url, mock_init @pytest.fixture() def performance_collector(performance_local_db, web3: Web3, frame_config: FrameConfig): - yield PerformanceCollector(web3) + yield PerformanceCollector(web3.cc) @pytest.fixture() def performance_web_server(performance_local_db): - Thread(target=serve, daemon=True).start() + Thread( + target=uvicorn.run, args=(app,), kwargs={'host': '127.0.0.1', 'port': 9020, 'log_level': 'error'}, daemon=True + ).start() yield @@ -91,7 +110,10 @@ def missed_initial_frame(frame_config: FrameConfig, cycle_iterations): @pytest.mark.fork @pytest.mark.parametrize( 'module', - [csm_module], + [ + csm_module, + # cm_module # TODO: uncomment when Curated Module is ready on Mainnet + ], indirect=True, ) @pytest.mark.parametrize( @@ -99,14 +121,14 @@ def missed_initial_frame(frame_config: FrameConfig, cycle_iterations): [start_before_initial_epoch, start_after_initial_epoch, missed_initial_frame], indirect=True, ) -def test_csm_module_report( +def test_staking_module_module_report( performance_web_server, performance_collector, module, set_oracle_members, running_finalized_slots, account_from ): assert module.report_contract.get_last_processing_ref_slot() == 0, "Last processing ref slot should be 0" members = set_oracle_members(count=2) report_frame = None - to_distribute_before_report = module.w3.csm.fee_distributor.shares_to_distribute() + to_distribute_before_report = module.w3.staking_module.fee_distributor.shares_to_distribute() switch_finalized, _ = running_finalized_slots # pylint:disable=duplicate-code @@ -120,13 +142,13 @@ def test_csm_module_report( module._receive_last_finalized_slot() # pylint: disable=protected-access ) - last_processing_after_report = module.w3.csm.oracle.get_last_processing_ref_slot() + last_processing_after_report = module.w3.staking_module.oracle.get_last_processing_ref_slot() assert ( last_processing_after_report == report_frame.ref_slot ), "Last processing ref slot should equal to initial ref slot" - to_distribute_after_report = module.w3.csm.fee_distributor.shares_to_distribute() + to_distribute_after_report = module.w3.staking_module.fee_distributor.shares_to_distribute() assert to_distribute_after_report < to_distribute_before_report, "Shares to distribute should decrease" - nos_count = int(module.w3.csm.module.functions.getNodeOperatorsCount().call()) + nos_count = int(module.w3.staking_module.module.functions.getNodeOperatorsCount().call()) assert to_distribute_after_report <= nos_count, "Dust after distribution should be less or equal to NOs count" diff --git a/tests/integration/contracts/test_accounting.py b/tests/integration/contracts/test_accounting.py index 463f2b783..b78819b93 100644 --- a/tests/integration/contracts/test_accounting.py +++ b/tests/integration/contracts/test_accounting.py @@ -1,7 +1,7 @@ import pytest from web3.types import Wei -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.types import ( ReportSimulationPayload, ReportSimulationResults, ) diff --git a/tests/integration/contracts/test_accounting_oracle.py b/tests/integration/contracts/test_accounting_oracle.py index 25fd8a12f..820795e28 100644 --- a/tests/integration/contracts/test_accounting_oracle.py +++ b/tests/integration/contracts/test_accounting_oracle.py @@ -1,7 +1,7 @@ import pytest from web3.contract.contract import ContractFunction -from src.modules.accounting.types import AccountingProcessingState +from src.modules.oracles.accounting.types import AccountingProcessingState from tests.integration.contracts.contract_utils import check_contract, check_is_instance_of diff --git a/tests/integration/contracts/test_csm_extension.py b/tests/integration/contracts/test_csm_extension.py index ea8fbdc9e..eb4fe52a4 100644 --- a/tests/integration/contracts/test_csm_extension.py +++ b/tests/integration/contracts/test_csm_extension.py @@ -2,22 +2,22 @@ import pytest -from src.web3py.extensions.csm import CSM +from src.web3py.extensions.staking_module import StakingModuleContracts from src.web3py.types import Web3 @pytest.fixture def w3(web3_provider_integration): - web3_provider_integration.attach_modules({"csm": CSM}) + web3_provider_integration.attach_modules({"staking_module": StakingModuleContracts}) return web3_provider_integration @pytest.mark.integration @pytest.mark.skip("CSM v2 is not yet live") def test_csm_extension(w3: Web3): - w3.csm.get_csm_last_processing_ref_slot(Mock(block_hash="latest")) - w3.csm.get_rewards_tree_root(Mock(block_hash="latest")) - w3.csm.get_rewards_tree_cid(Mock(block_hash="latest")) - w3.csm.get_curve_params(Mock(0), Mock(block_hash="latest")) - w3.csm.get_strikes_tree_root(Mock(block_hash="latest")) - w3.csm.get_strikes_tree_cid(Mock(block_hash="latest")) + w3.staking_module.get_last_processing_ref_slot(Mock(block_hash="latest")) + w3.staking_module.get_rewards_tree_root(Mock(block_hash="latest")) + w3.staking_module.get_rewards_tree_cid(Mock(block_hash="latest")) + w3.staking_module.get_curve_params(Mock(0), Mock(block_hash="latest")) + w3.staking_module.get_strikes_tree_root(Mock(block_hash="latest")) + w3.staking_module.get_strikes_tree_cid(Mock(block_hash="latest")) diff --git a/tests/integration/contracts/test_lazy_oracle.py b/tests/integration/contracts/test_lazy_oracle.py index 930fa6a3c..85991f980 100644 --- a/tests/integration/contracts/test_lazy_oracle.py +++ b/tests/integration/contracts/test_lazy_oracle.py @@ -1,6 +1,6 @@ import pytest -from src.modules.accounting.types import OnChainIpfsVaultReportData +from src.modules.oracles.accounting.types import OnChainIpfsVaultReportData from tests.integration.contracts.contract_utils import check_contract, check_value_type diff --git a/tests/integration/contracts/test_lido.py b/tests/integration/contracts/test_lido.py index 05f6452bf..af5d4bd0d 100644 --- a/tests/integration/contracts/test_lido.py +++ b/tests/integration/contracts/test_lido.py @@ -1,6 +1,6 @@ import pytest -from src.modules.accounting.types import BeaconStat +from src.modules.oracles.accounting.types import BeaconStat from tests.integration.contracts.contract_utils import check_contract, check_value_type diff --git a/tests/integration/contracts/test_oracle_report_sanity_checker.py b/tests/integration/contracts/test_oracle_report_sanity_checker.py index 4f9d8c582..d2113f164 100644 --- a/tests/integration/contracts/test_oracle_report_sanity_checker.py +++ b/tests/integration/contracts/test_oracle_report_sanity_checker.py @@ -1,6 +1,6 @@ import pytest -from src.modules.accounting.types import OracleReportLimits +from src.modules.oracles.accounting.types import OracleReportLimits from tests.integration.contracts.contract_utils import check_contract, check_is_instance_of diff --git a/tests/integration/contracts/test_validator_exit_bus_oracle.py b/tests/integration/contracts/test_validator_exit_bus_oracle.py index 2fff95c69..3b3f84920 100644 --- a/tests/integration/contracts/test_validator_exit_bus_oracle.py +++ b/tests/integration/contracts/test_validator_exit_bus_oracle.py @@ -1,6 +1,6 @@ import pytest -from src.modules.ejector.types import EjectorProcessingState +from src.modules.oracles.ejector.types import EjectorProcessingState from tests.integration.contracts.contract_utils import ( check_contract, check_is_instance_of, diff --git a/tests/integration/contracts/test_withdrawal_queue_nft_contract.py b/tests/integration/contracts/test_withdrawal_queue_nft_contract.py index f738f891d..9d4f79ccc 100644 --- a/tests/integration/contracts/test_withdrawal_queue_nft_contract.py +++ b/tests/integration/contracts/test_withdrawal_queue_nft_contract.py @@ -1,6 +1,6 @@ import pytest -from src.modules.accounting.types import BatchState, WithdrawalRequestStatus +from src.modules.oracles.accounting.types import BatchState, WithdrawalRequestStatus from tests.integration.contracts.contract_utils import check_contract, check_is_instance_of diff --git a/tests/modules/accounting/bunker/conftest.py b/tests/modules/accounting/bunker/conftest.py index 674b9dbc3..6173b8d8d 100644 --- a/tests/modules/accounting/bunker/conftest.py +++ b/tests/modules/accounting/bunker/conftest.py @@ -3,7 +3,7 @@ import pytest from src.constants import FAR_FUTURE_EPOCH -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker import BunkerService from src.providers.keys.types import LidoKey diff --git a/tests/modules/accounting/bunker/test_bunker.py b/tests/modules/accounting/bunker/test_bunker.py index 8fd63fa7b..240e317e4 100644 --- a/tests/modules/accounting/bunker/test_bunker.py +++ b/tests/modules/accounting/bunker/test_bunker.py @@ -4,7 +4,7 @@ import pytest from web3.types import Wei -from src.modules.accounting.types import ReportSimulationFeeDistribution, ReportSimulationResults +from src.modules.oracles.accounting.types import ReportSimulationFeeDistribution, ReportSimulationResults from src.providers.consensus.types import BeaconStateView from src.services.bunker import BunkerService from src.types import ReferenceBlockStamp diff --git a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py index 30d2626bc..3109ab1fe 100644 --- a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py +++ b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py @@ -6,8 +6,8 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, EPOCHS_PER_SLASHINGS_VECTOR, ) -from src.modules.submodules.consensus import FrameConfig -from src.modules.submodules.types import ChainConfig +from src.modules.oracles.common.consensus import FrameConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty from src.types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex diff --git a/tests/modules/accounting/staking_vault/test_ifps_report.py b/tests/modules/accounting/staking_vault/test_ifps_report.py index 73ff14aaa..4ba3c19af 100644 --- a/tests/modules/accounting/staking_vault/test_ifps_report.py +++ b/tests/modules/accounting/staking_vault/test_ifps_report.py @@ -2,7 +2,7 @@ from eth_typing import HexStr from src.services.staking_vaults import StakingVaultsService -from src.modules.accounting.types import StakingVaultIpfsReport +from src.modules.oracles.accounting.types import StakingVaultIpfsReport from src.providers.ipfs import CIDv0 from src.types import SlotNumber, FrameNumber from src.utils.slot import get_blockstamp diff --git a/tests/modules/accounting/staking_vault/test_staking_vaults.py b/tests/modules/accounting/staking_vault/test_staking_vaults.py index 39c81771d..695409691 100644 --- a/tests/modules/accounting/staking_vault/test_staking_vaults.py +++ b/tests/modules/accounting/staking_vault/test_staking_vaults.py @@ -9,7 +9,7 @@ from web3.types import Timestamp, Wei from src.constants import FAR_FUTURE_EPOCH, TOTAL_BASIS_POINTS -from src.modules.accounting.events import ( +from src.modules.oracles.accounting.events import ( BadDebtSocializedEvent, BadDebtWrittenOffToBeInternalizedEvent, BurnedSharesOnVaultEvent, @@ -20,7 +20,7 @@ VaultRebalancedEvent, sort_events, ) -from src.modules.accounting.types import ( +from src.modules.oracles.accounting.types import ( ExtraValue, MerkleValue, OnChainIpfsVaultReportData, @@ -33,7 +33,7 @@ VaultsMap, VaultTotalValueMap, ) -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.common.types import ChainConfig, FrameConfig from src.providers.consensus.types import ( BeaconBlockBody, BlockDetailsResponse, @@ -2261,7 +2261,7 @@ def test_get_start_point_fresh_devnet_case(self): ipfs_data = OnChainIpfsVaultReportData( timestamp=1690000100, ref_slot=SlotNumber(1230), tree_root=b'\xab\xcd\xef', report_cid="" - ) # важно! + ) # Important! frame_config = FrameConfig(initial_epoch=10, epochs_per_frame=2, fast_lane_length_slots=16) diff --git a/tests/modules/accounting/test_accounting_module.py b/tests/modules/accounting/test_accounting_module.py index 062dca112..cdfcf347b 100644 --- a/tests/modules/accounting/test_accounting_module.py +++ b/tests/modules/accounting/test_accounting_module.py @@ -7,10 +7,10 @@ from web3.types import Wei from src import variables -from src.modules.accounting import accounting as accounting_module -from src.modules.accounting.accounting import Accounting, logger as accounting_logger -from src.modules.accounting.third_phase.types import FormatList -from src.modules.accounting.types import ( +from src.modules.oracles.accounting import accounting as accounting_module +from src.modules.oracles.accounting.accounting import Accounting, logger as accounting_logger +from src.modules.oracles.accounting.third_phase.types import FormatList +from src.modules.oracles.accounting.types import ( AccountingProcessingState, ReportSimulationFeeDistribution, ReportSimulationPayload, @@ -20,8 +20,8 @@ VaultsMap, VaultTreeNode, ) -from src.modules.submodules.oracle_module import ModuleExecuteDelay -from src.modules.submodules.types import ( +from src.modules.oracles.common.oracle_module import ModuleExecuteDelay +from src.modules.common.types import ( ZERO_HASH, ChainConfig, CurrentFrame, diff --git a/tests/modules/accounting/test_extra_data.py b/tests/modules/accounting/test_extra_data.py index f26d4b60f..3e74dab08 100644 --- a/tests/modules/accounting/test_extra_data.py +++ b/tests/modules/accounting/test_extra_data.py @@ -1,9 +1,9 @@ import pytest from web3 import Web3 -from src.modules.accounting.third_phase.extra_data import ExtraDataService, ItemPayload -from src.modules.accounting.third_phase.types import FormatList -from src.modules.submodules.types import ZERO_HASH +from src.modules.oracles.accounting.third_phase.extra_data import ExtraDataService, ItemPayload +from src.modules.oracles.accounting.third_phase.types import FormatList +from src.modules.common.types import ZERO_HASH @pytest.mark.unit diff --git a/tests/modules/accounting/test_safe_border_unit.py b/tests/modules/accounting/test_safe_border_unit.py index bab6435c4..f547c040b 100644 --- a/tests/modules/accounting/test_safe_border_unit.py +++ b/tests/modules/accounting/test_safe_border_unit.py @@ -3,7 +3,7 @@ import pytest -from src.modules.submodules.consensus import ChainConfig, FrameConfig +from src.modules.oracles.common.consensus import ChainConfig, FrameConfig from src.providers.consensus.types import ValidatorState from src.services.safe_border import SafeBorder from src.web3py.extensions.lido_validators import Validator diff --git a/tests/modules/accounting/test_validator_state.py b/tests/modules/accounting/test_validator_state.py index c8962a1b0..71272583f 100644 --- a/tests/modules/accounting/test_validator_state.py +++ b/tests/modules/accounting/test_validator_state.py @@ -5,7 +5,7 @@ from eth_typing import HexStr from src.constants import FAR_FUTURE_EPOCH -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey from src.services.validator_state import LidoValidatorStateService diff --git a/tests/modules/accounting/test_withdrawal_integration.py b/tests/modules/accounting/test_withdrawal_integration.py index 78f131565..5a6079a88 100644 --- a/tests/modules/accounting/test_withdrawal_integration.py +++ b/tests/modules/accounting/test_withdrawal_integration.py @@ -2,7 +2,7 @@ from eth_typing import BlockNumber from web3.types import Timestamp -from src.modules.submodules.types import FrameConfig, ChainConfig +from src.modules.common.types import FrameConfig, ChainConfig from src.services.withdrawal import Withdrawal from src.constants import SHARE_RATE_PRECISION_E27 from src.types import ReferenceBlockStamp, SlotNumber, EpochNumber diff --git a/tests/modules/accounting/test_withdrawal_unit.py b/tests/modules/accounting/test_withdrawal_unit.py index f1e66d703..fc818d78b 100644 --- a/tests/modules/accounting/test_withdrawal_unit.py +++ b/tests/modules/accounting/test_withdrawal_unit.py @@ -2,9 +2,9 @@ from unittest.mock import Mock from src.services.withdrawal import Withdrawal -from src.modules.submodules.consensus import ChainConfig, FrameConfig +from src.modules.oracles.common.consensus import ChainConfig, FrameConfig from src.constants import SHARE_RATE_PRECISION_E27 -from src.modules.accounting.types import BatchState +from src.modules.oracles.accounting.types import BatchState from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.configs import OracleReportLimitsFactory diff --git a/tests/modules/csm/conftest.py b/tests/modules/csm/conftest.py new file mode 100644 index 000000000..36d19b8eb --- /dev/null +++ b/tests/modules/csm/conftest.py @@ -0,0 +1,17 @@ +import pytest + +from unittest.mock import patch + +from src import variables +from src.web3py.extensions.staking_module import StakingModuleContracts + + +@pytest.fixture() +def web3(web3): + with patch.object(variables, 'CURATED_MODULE_ADDRESS', None): + web3.attach_modules( + { + "staking_module": StakingModuleContracts, + } + ) + yield web3 diff --git a/tests/modules/csm/test_csm_distribution.py b/tests/modules/csm/test_csm_distribution.py index 649122237..9b435b691 100644 --- a/tests/modules/csm/test_csm_distribution.py +++ b/tests/modules/csm/test_csm_distribution.py @@ -12,10 +12,14 @@ MIN_ACTIVATION_BALANCE, EFFECTIVE_BALANCE_INCREMENT, ) -from src.modules.csm.distribution import Distribution, ValidatorDuties, ValidatorDutiesOutcome -from src.modules.csm.log import FramePerfLog, ValidatorFrameSummary, OperatorFrameSummary -from src.modules.csm.state import DutyAccumulator, State, NetworkDuties, Frame -from src.modules.csm.types import StrikesList +from src.modules.oracles.staking_modules.common.distribution import ( + Distribution, + ValidatorDuties, + ValidatorDutiesOutcome, +) +from src.modules.oracles.staking_modules.common.log import FramePerfLog, ValidatorFrameSummary, OperatorFrameSummary +from src.modules.oracles.staking_modules.common.state import DutyAccumulator, State, NetworkDuties, Frame +from src.modules.oracles.staking_modules.common.types import StrikesList from src.providers.execution.contracts.cs_fee_distributor import CSFeeDistributorContract from src.providers.execution.contracts.cs_parameters_registry import ( StrikesParams, @@ -26,7 +30,7 @@ ) from src.providers.execution.exceptions import InconsistentData from src.types import NodeOperatorId, EpochNumber, ValidatorIndex -from src.web3py.extensions import CSM +from src.web3py.extensions import StakingModuleContracts from src.web3py.types import Web3 from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.no_registry import LidoValidatorFactory, ValidatorStateFactory @@ -297,11 +301,13 @@ def test_calculate_distribution( expected_strikes, ): # Mocking the data from EL - w3 = Mock(spec=Web3, csm=Mock(spec=CSM, fee_distributor=Mock(spec=CSFeeDistributorContract))) - w3.csm.fee_distributor.shares_to_distribute = Mock(side_effect=shares_to_distribute) - w3.csm.get_curve_params = mocked_curve_params + w3 = Mock( + spec=Web3, staking_module=Mock(spec=StakingModuleContracts, fee_distributor=Mock(spec=CSFeeDistributorContract)) + ) + w3.staking_module.fee_distributor.shares_to_distribute = Mock(side_effect=shares_to_distribute) + w3.staking_module.get_curve_params = mocked_curve_params - distribution = Distribution(w3, converter=..., state=State()) + distribution = Distribution(w3, converter=..., state=State(oracle_name='test')) distribution._get_module_validators = Mock(...) distribution.state.data = {f: {} for f in frames} distribution._get_frame_blockstamp = Mock(side_effect=frame_blockstamps) @@ -323,11 +329,13 @@ def test_calculate_distribution( @pytest.mark.unit def test_calculate_distribution_handles_invalid_distribution(): # Mocking the data from EL - w3 = Mock(spec=Web3, csm=Mock(spec=CSM, fee_distributor=Mock(spec=CSFeeDistributorContract))) - w3.csm.fee_distributor.shares_to_distribute = Mock(return_value=500) - w3.csm.get_curve_params = Mock(...) + w3 = Mock( + spec=Web3, staking_module=Mock(spec=StakingModuleContracts, fee_distributor=Mock(spec=CSFeeDistributorContract)) + ) + w3.staking_module.fee_distributor.shares_to_distribute = Mock(return_value=500) + w3.staking_module.get_curve_params = Mock(...) - distribution = Distribution(w3, converter=..., state=State()) + distribution = Distribution(w3, converter=..., state=State(oracle_name='test')) distribution._get_module_validators = Mock(...) distribution.state.data = {(EpochNumber(0), EpochNumber(31)): {}} distribution._get_frame_blockstamp = Mock(return_value=ReferenceBlockStampFactory.build(ref_epoch=31)) @@ -351,11 +359,13 @@ def test_calculate_distribution_handles_invalid_distribution(): @pytest.mark.unit def test_calculate_distribution_handles_invalid_distribution_in_total(): # Mocking the data from EL - w3 = Mock(spec=Web3, csm=Mock(spec=CSM, fee_distributor=Mock(spec=CSFeeDistributorContract))) - w3.csm.fee_distributor.shares_to_distribute = Mock(return_value=500) - w3.csm.get_curve_params = Mock(...) + w3 = Mock( + spec=Web3, staking_module=Mock(spec=StakingModuleContracts, fee_distributor=Mock(spec=CSFeeDistributorContract)) + ) + w3.staking_module.fee_distributor.shares_to_distribute = Mock(return_value=500) + w3.staking_module.get_curve_params = Mock(...) - distribution = Distribution(w3, converter=..., state=State()) + distribution = Distribution(w3, converter=..., state=State(oracle_name='test')) distribution._get_module_validators = Mock(...) distribution.state.data = {(EpochNumber(0), EpochNumber(31)): {}} distribution._get_frame_blockstamp = Mock(return_value=ReferenceBlockStampFactory.build(ref_epoch=31)) @@ -835,11 +845,11 @@ def test_calculate_distribution_in_frame( ): log = FramePerfLog(blockstamp=..., frame=...) # Mocking the data from EL - w3 = Mock(spec=Web3, csm=Mock(spec=CSM)) - w3.csm.get_curve_params = mocked_curve_params + w3 = Mock(spec=Web3, staking_module=Mock(spec=StakingModuleContracts)) + w3.staking_module.get_curve_params = mocked_curve_params frame = (EpochNumber(0), EpochNumber(31)) - state = State() + state = State(oracle_name='test') state.migrate(*frame, epochs_per_frame=32) state.data = {frame: frame_state_data} @@ -1120,7 +1130,7 @@ def test_merge_strikes( expected: dict, ): distribution = Distribution(Mock(csm=Mock()), Mock(), Mock()) - distribution.w3.csm.get_curve_params = Mock( + distribution.w3.staking_module.get_curve_params = Mock( side_effect=lambda no_id, _: Mock(strikes_params=threshold_per_op[no_id]) ) @@ -1271,10 +1281,10 @@ def test_get_validator_duties_outcome_scales_by_effective_balance(multiplier: in @pytest.mark.unit def test_calculate_distribution_in_frame_assigns_keys_by_sorted_order(): - w3 = Mock(spec=Web3, csm=Mock()) + w3 = Mock(spec=Web3, staking_module=Mock()) reward_share_data = Mock() reward_share_data.get_for = Mock(side_effect=lambda k: {1: 1.0, 2: 0.9, 3: 0.8, 4: 0.7, 5: 0.6, 6: 0.5}[k]) - w3.csm.get_curve_params = Mock( + w3.staking_module.get_curve_params = Mock( return_value=CurveParams( strikes_params=..., perf_leeway_data=Mock(get_for=Mock(return_value=0.0)), @@ -1283,7 +1293,7 @@ def test_calculate_distribution_in_frame_assigns_keys_by_sorted_order(): ) ) - distribution = Distribution(w3, converter=..., state=State()) + distribution = Distribution(w3, converter=..., state=State(oracle_name='test')) distribution._get_network_performance = Mock(return_value=0.9) frame = (EpochNumber(0), EpochNumber(31)) diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index 66c1d048a..17bf309a2 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -7,16 +7,18 @@ import pytest from hexbytes import HexBytes -from src.constants import UINT64_MAX, CSM_LOGS_VERSION -from src.modules.csm.csm import CSMError, CSOracle, LastReport -from src.modules.csm.distribution import Distribution -from src.modules.csm.log import FramePerfLog, Logs -from src.modules.csm.state import State -from src.modules.csm.tree import RewardsTree, StrikesTree -from src.modules.csm.types import StrikesList -from src.modules.performance.common.db import Duty -from src.modules.submodules.oracle_module import ModuleExecuteDelay -from src.modules.submodules.types import ZERO_HASH, CurrentFrame +from src.constants import UINT64_MAX, STAKING_MODULE_LOGS_VERSION +from src.modules.oracles.staking_modules.base import SMPerformanceOracleError +from src.modules.oracles.staking_modules.community_staking.csm import CSPerformanceOracle +from src.modules.oracles.staking_modules.common.helpers.last_report import LastReport +from src.modules.oracles.staking_modules.common.distribution import Distribution +from src.modules.oracles.staking_modules.common.log import Logs +from src.modules.oracles.staking_modules.common.state import State +from src.modules.oracles.staking_modules.common.tree import RewardsTree, StrikesTree +from src.modules.oracles.staking_modules.common.types import StrikesList +from src.modules.sidecars.performance.common.db import Duty +from src.modules.oracles.common.oracle_module import ModuleExecuteDelay +from src.modules.common.types import ZERO_HASH, CurrentFrame from src.providers.consensus.types import Validator, ValidatorState from src.providers.execution.exceptions import InconsistentData from src.providers.ipfs import CID @@ -34,11 +36,11 @@ def mock_load_state(monkeypatch: pytest.MonkeyPatch): @pytest.fixture() def module(web3): - yield CSOracle(web3) + yield CSPerformanceOracle(web3) @pytest.mark.unit -def test_init(module: CSOracle): +def test_init(module: CSPerformanceOracle): assert module @@ -71,7 +73,7 @@ def make_validator(index: int, activation_epoch: int = 0, exit_epoch: int = 100) @pytest.fixture() -def mock_chain_config(module: CSOracle): +def mock_chain_config(module: CSPerformanceOracle): module.get_chain_config = Mock( return_value=ChainConfigFactory.build( slots_per_epoch=32, @@ -207,14 +209,14 @@ class FrameTestParam: last_processing_ref_slot=0, current_ref_slot=last_slot_of_epoch(1), finalized_slot=last_slot_of_epoch(1), - expected_frame=CSMError, + expected_frame=SMPerformanceOracleError, ), id="negative_first_frame", ), ], ) @pytest.mark.unit -def test_current_frame_range(module: CSOracle, mock_chain_config: NoReturn, param: FrameTestParam): +def test_current_frame_range(module: CSPerformanceOracle, mock_chain_config: NoReturn, param: FrameTestParam): module.get_frame_config = Mock( return_value=FrameConfigFactory.build( initial_epoch=slot_to_epoch(param.initial_ref_slot), @@ -223,7 +225,7 @@ def test_current_frame_range(module: CSOracle, mock_chain_config: NoReturn, para ) ) - module.w3.csm.get_csm_last_processing_ref_slot = Mock(return_value=param.last_processing_ref_slot) + module.w3.staking_module.get_last_processing_ref_slot = Mock(return_value=param.last_processing_ref_slot) module.get_initial_or_current_frame = Mock( return_value=CurrentFrame( ref_slot=SlotNumber(param.current_ref_slot), @@ -246,7 +248,7 @@ def test_current_frame_range(module: CSOracle, mock_chain_config: NoReturn, para @pytest.mark.unit -def test_set_epochs_range_to_collect_posts_new_demand(module: CSOracle, mock_chain_config: NoReturn): +def test_set_epochs_range_to_collect_posts_new_demand(module: CSPerformanceOracle, mock_chain_config: NoReturn): blockstamp = ReferenceBlockStampFactory.build() module.state = Mock(migrate=Mock(), log_progress=Mock()) converter = Mock() @@ -264,11 +266,13 @@ def test_set_epochs_range_to_collect_posts_new_demand(module: CSOracle, mock_cha module.state.log_progress.assert_called_once() module.w3.performance.is_range_available.assert_called_once_with(10, 20) module.w3.performance.get_epochs_demand.assert_called_once() - module.w3.performance.post_epochs_demand.assert_called_once_with("CSOracle", 10, 20) + module.w3.performance.post_epochs_demand.assert_called_once_with("CSPerformanceOracle", 10, 20) @pytest.mark.unit -def test_set_epochs_range_to_collect_skips_post_when_demand_same(module: CSOracle, mock_chain_config: NoReturn): +def test_set_epochs_range_to_collect_skips_post_when_demand_same( + module: CSPerformanceOracle, mock_chain_config: NoReturn +): blockstamp = ReferenceBlockStampFactory.build() module.state = Mock(migrate=Mock(), log_progress=Mock()) converter = Mock() @@ -276,7 +280,7 @@ def test_set_epochs_range_to_collect_skips_post_when_demand_same(module: CSOracl module.converter = Mock(return_value=converter) module.get_epochs_range_to_process = Mock(return_value=(10, 20)) module.w3 = Mock() - module.w3.performance.get_epochs_demands = Mock(return_value={"CSOracle": (10, 20)}) + module.w3.performance.get_epochs_demands = Mock(return_value={"CSPerformanceOracle": (10, 20)}) module.w3.performance.post_epochs_demand = Mock() module.set_epochs_range_to_collect(blockstamp) @@ -287,7 +291,7 @@ def test_set_epochs_range_to_collect_skips_post_when_demand_same(module: CSOracl @pytest.fixture() -def mock_frame_config(module: CSOracle): +def mock_frame_config(module: CSPerformanceOracle): module.get_frame_config = Mock( return_value=FrameConfigFactory.build( initial_epoch=0, @@ -351,7 +355,7 @@ class CollectDataCase: ) @pytest.mark.unit def test_collect_data_handles_range_availability( - module: CSOracle, mock_chain_config: NoReturn, mock_frame_config: NoReturn, caplog, case: CollectDataCase + module: CSPerformanceOracle, mock_chain_config: NoReturn, mock_frame_config: NoReturn, caplog, case: CollectDataCase ): module.w3 = Mock() module.w3.performance.is_range_available = Mock(return_value=case.range_available) @@ -375,7 +379,7 @@ def test_collect_data_handles_range_availability( @pytest.mark.unit -def test_fulfill_state_handles_epoch_data(module: CSOracle): +def test_fulfill_state_handles_epoch_data(module: CSPerformanceOracle): module._receive_last_finalized_slot = Mock(return_value="finalized") validator_a = make_validator(0, activation_epoch=0, exit_epoch=10) validator_b = make_validator(1, activation_epoch=0, exit_epoch=10) @@ -449,7 +453,7 @@ def test_fulfill_state_handles_epoch_data(module: CSOracle): @pytest.mark.unit -def test_fulfill_state_raises_on_inactive_missed_attestation(module: CSOracle): +def test_fulfill_state_raises_on_inactive_missed_attestation(module: CSPerformanceOracle): inactive_validator = make_validator(5, activation_epoch=10, exit_epoch=20) module._receive_last_finalized_slot = Mock(return_value="finalized") module.w3 = Mock() @@ -483,7 +487,7 @@ def test_fulfill_state_raises_on_inactive_missed_attestation(module: CSOracle): @pytest.mark.unit -def test_validate_state_uses_ref_epoch(module: CSOracle): +def test_validate_state_uses_ref_epoch(module: CSPerformanceOracle): blockstamp = ReferenceBlockStampFactory.build(ref_epoch=123) module.get_epochs_range_to_process = Mock(return_value=(5, 10)) module.state = Mock(validate=Mock()) @@ -502,10 +506,10 @@ def test_validate_state_uses_ref_epoch(module: CSOracle): ], ) @pytest.mark.unit -def test_is_main_data_submitted(module: CSOracle, last_ref_slot: int, current_ref_slot: int, expected: bool): +def test_is_main_data_submitted(module: CSPerformanceOracle, last_ref_slot: int, current_ref_slot: int, expected: bool): blockstamp = ReferenceBlockStampFactory.build() module.w3 = Mock() - module.w3.csm.get_csm_last_processing_ref_slot = Mock(return_value=SlotNumber(last_ref_slot)) + module.w3.staking_module.get_last_processing_ref_slot = Mock(return_value=SlotNumber(last_ref_slot)) module.get_initial_or_current_frame = Mock( return_value=CurrentFrame( ref_slot=SlotNumber(current_ref_slot), @@ -518,7 +522,7 @@ def test_is_main_data_submitted(module: CSOracle, last_ref_slot: int, current_re @pytest.mark.parametrize("submitted", [True, False]) @pytest.mark.unit -def test_is_contract_reportable_relies_on_is_main_data_submitted(module: CSOracle, submitted: bool): +def test_is_contract_reportable_relies_on_is_main_data_submitted(module: CSPerformanceOracle, submitted: bool): module.is_main_data_submitted = Mock(return_value=submitted) result = module.is_contract_reportable(ReferenceBlockStampFactory.build()) @@ -528,7 +532,7 @@ def test_is_contract_reportable_relies_on_is_main_data_submitted(module: CSOracl @pytest.mark.unit -def test_publish_tree_uploads_encoded_tree(module: CSOracle): +def test_publish_tree_uploads_encoded_tree(module: CSPerformanceOracle): tree = Mock() tree.encode.return_value = b"tree" module.w3 = Mock() @@ -541,7 +545,7 @@ def test_publish_tree_uploads_encoded_tree(module: CSOracle): @pytest.mark.unit -def test_publish_log_uploads_encoded_log(module: CSOracle, monkeypatch: pytest.MonkeyPatch): +def test_publish_log_uploads_encoded_log(module: CSPerformanceOracle, monkeypatch: pytest.MonkeyPatch): logs = Logs() logs.frames = [Mock()] encode_mock = Mock(return_value=b"log") @@ -749,12 +753,10 @@ class BuildReportTestParam: ], ) @pytest.mark.unit -def test_build_report(module: CSOracle, param: BuildReportTestParam): +def test_build_report(module: CSPerformanceOracle, param: BuildReportTestParam): module.validate_state = Mock() module.report_contract.get_consensus_version = Mock(return_value=1) - module._get_last_report = Mock(return_value=param.last_report) - # mock current frame - module.calculate_distribution = param.curr_distribution + module.calculate_distribution = Mock(return_value=(param.curr_distribution(), param.last_report)) module.make_rewards_tree = Mock(return_value=Mock(root=param.curr_rewards_tree_root)) module.make_strikes_tree = Mock(return_value=Mock(root=param.curr_strikes_tree_root)) module.publish_tree = Mock( @@ -770,11 +772,11 @@ def test_build_report(module: CSOracle, param: BuildReportTestParam): assert module.make_rewards_tree.call_args == param.expected_make_rewards_tree_call_args assert report == param.expected_func_result - assert module.publish_log.call_args[0][0]._ver == CSM_LOGS_VERSION + assert module.publish_log.call_args[0][0]._ver == STAKING_MODULE_LOGS_VERSION @pytest.mark.unit -def test_execute_module_not_collected(module: CSOracle): +def test_execute_module_not_collected(module: CSPerformanceOracle): module._check_compatability = Mock(return_value=True) module.get_blockstamp_for_report = Mock(return_value=Mock(slot_number=100500)) module.set_epochs_range_to_collect = Mock() @@ -787,7 +789,7 @@ def test_execute_module_not_collected(module: CSOracle): @pytest.mark.unit -def test_execute_module_skips_collecting_if_forward_compatible(module: CSOracle): +def test_execute_module_skips_collecting_if_forward_compatible(module: CSPerformanceOracle): module._check_compatability = Mock(return_value=False) module.collect_data = Mock(return_value=False) @@ -799,7 +801,7 @@ def test_execute_module_skips_collecting_if_forward_compatible(module: CSOracle) @pytest.mark.unit -def test_execute_module_no_report_blockstamp(module: CSOracle): +def test_execute_module_no_report_blockstamp(module: CSPerformanceOracle): module._check_compatability = Mock(return_value=True) module.set_epochs_range_to_collect = Mock() module.collect_data = Mock(return_value=True) @@ -812,7 +814,7 @@ def test_execute_module_no_report_blockstamp(module: CSOracle): @pytest.mark.unit -def test_execute_module_processed(module: CSOracle): +def test_execute_module_processed(module: CSPerformanceOracle): module.set_epochs_range_to_collect = Mock() module.collect_data = Mock(return_value=True) module.get_blockstamp_for_report = Mock(return_value=Mock(slot_number=100500)) @@ -826,31 +828,36 @@ def test_execute_module_processed(module: CSOracle): @pytest.mark.unit -def test_calculate_distribution_lru_cache(module: CSOracle): +def test_calculate_distribution_lru_cache(module: CSPerformanceOracle): blockstamp = Mock() last_report = Mock() + last_report.strikes = {} # Create proper dictionary instead of Mock + last_report.rewards = [] # Add empty list instead of Mock for rewards mock_distribution_result = Mock() - with patch('src.modules.csm.csm.Distribution') as MockDistribution: + with patch('src.modules.oracles.staking_modules.base.Distribution') as MockDistribution: mock_distribution_instance = MockDistribution.return_value mock_distribution_instance.calculate.return_value = mock_distribution_result module.converter = Mock() module.state = Mock() + module._get_last_report = Mock(return_value=last_report) - result1 = module.calculate_distribution(blockstamp, last_report) + result1, last_report1 = module.calculate_distribution(blockstamp) - result2 = module.calculate_distribution(blockstamp, last_report) + result2, last_report2 = module.calculate_distribution(blockstamp) assert result1 is result2 + assert last_report1 is last_report2 assert result1 is mock_distribution_result + assert last_report1 is last_report assert MockDistribution.call_count == 1 assert mock_distribution_instance.calculate.call_count == 1 module.calculate_distribution.cache_clear() - result3 = module.calculate_distribution(blockstamp, last_report) + result3, last_report3 = module.calculate_distribution(blockstamp) assert MockDistribution.call_count == 2 assert result3 is mock_distribution_result @@ -868,8 +875,8 @@ class RewardsTreeTestParam: pytest.param(RewardsTreeTestParam(shares={}, expected_tree_values=ValueError), id="empty"), ], ) -def test_make_rewards_tree_negative(module: CSOracle, param: RewardsTreeTestParam): - module.w3.csm.module.MAX_OPERATORS_COUNT = UINT64_MAX +def test_make_rewards_tree_negative(module: CSPerformanceOracle, param: RewardsTreeTestParam): + module.w3.staking_module.module.MAX_OPERATORS_COUNT = UINT64_MAX with pytest.raises(ValueError): module.make_rewards_tree(param.shares) @@ -918,8 +925,8 @@ def test_make_rewards_tree_negative(module: CSOracle, param: RewardsTreeTestPara ], ) @pytest.mark.unit -def test_make_rewards_tree(module: CSOracle, param: RewardsTreeTestParam): - module.w3.csm.module.MAX_OPERATORS_COUNT = UINT64_MAX +def test_make_rewards_tree(module: CSPerformanceOracle, param: RewardsTreeTestParam): + module.w3.staking_module.module.MAX_OPERATORS_COUNT = UINT64_MAX tree = module.make_rewards_tree(param.shares) assert tree.values == param.expected_tree_values @@ -938,8 +945,8 @@ class StrikesTreeTestParam: ], ) @pytest.mark.unit -def test_make_strikes_tree_negative(module: CSOracle, param: StrikesTreeTestParam): - module.w3.csm.module.MAX_OPERATORS_COUNT = UINT64_MAX +def test_make_strikes_tree_negative(module: CSPerformanceOracle, param: StrikesTreeTestParam): + module.w3.staking_module.module.MAX_OPERATORS_COUNT = UINT64_MAX with pytest.raises(ValueError): module.make_strikes_tree(param.strikes) @@ -977,8 +984,8 @@ def test_make_strikes_tree_negative(module: CSOracle, param: StrikesTreeTestPara ], ) @pytest.mark.unit -def test_make_strikes_tree(module: CSOracle, param: StrikesTreeTestParam): - module.w3.csm.module.MAX_OPERATORS_COUNT = UINT64_MAX +def test_make_strikes_tree(module: CSPerformanceOracle, param: StrikesTreeTestParam): + module.w3.staking_module.module.MAX_OPERATORS_COUNT = UINT64_MAX tree = module.make_strikes_tree(param.strikes) assert tree.values == param.expected_tree_values @@ -989,17 +996,17 @@ class TestLastReport: def test_load(self, web3: Web3): blockstamp = Mock() - web3.csm.get_rewards_tree_root = Mock(return_value=HexBytes(b"42")) - web3.csm.get_rewards_tree_cid = Mock(return_value=CID("QmRT")) - web3.csm.get_strikes_tree_root = Mock(return_value=HexBytes(b"17")) - web3.csm.get_strikes_tree_cid = Mock(return_value=CID("QmST")) + web3.staking_module.get_rewards_tree_root = Mock(return_value=HexBytes(b"42")) + web3.staking_module.get_rewards_tree_cid = Mock(return_value=CID("QmRT")) + web3.staking_module.get_strikes_tree_root = Mock(return_value=HexBytes(b"17")) + web3.staking_module.get_strikes_tree_cid = Mock(return_value=CID("QmST")) last_report = LastReport.load(web3, blockstamp, FrameNumber(0)) - web3.csm.get_rewards_tree_root.assert_called_once_with(blockstamp) - web3.csm.get_rewards_tree_cid.assert_called_once_with(blockstamp) - web3.csm.get_strikes_tree_root.assert_called_once_with(blockstamp) - web3.csm.get_strikes_tree_cid.assert_called_once_with(blockstamp) + web3.staking_module.get_rewards_tree_root.assert_called_once_with(blockstamp) + web3.staking_module.get_rewards_tree_cid.assert_called_once_with(blockstamp) + web3.staking_module.get_strikes_tree_root.assert_called_once_with(blockstamp) + web3.staking_module.get_strikes_tree_cid.assert_called_once_with(blockstamp) assert last_report.rewards_tree_root == HexBytes(b"42") assert last_report.rewards_tree_cid == CID("QmRT") diff --git a/tests/modules/csm/test_log.py b/tests/modules/csm/test_log.py index cf02ec334..f026cb307 100644 --- a/tests/modules/csm/test_log.py +++ b/tests/modules/csm/test_log.py @@ -1,8 +1,9 @@ import json import pytest -from src.constants import CSM_LOGS_VERSION -from src.modules.csm.log import FramePerfLog, DutyAccumulator, Logs +from src.constants import STAKING_MODULE_LOGS_VERSION +from src.modules.oracles.staking_modules.common.log import FramePerfLog, Logs +from src.modules.oracles.staking_modules.common.state import DutyAccumulator from src.providers.execution.contracts.cs_parameters_registry import PerformanceCoefficients from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/csm/test_state.py b/tests/modules/csm/test_state.py index a391c23ab..a850e46c0 100644 --- a/tests/modules/csm/test_state.py +++ b/tests/modules/csm/test_state.py @@ -7,15 +7,15 @@ import pytest from src import variables -from src.constants import CSM_STATE_VERSION -from src.modules.csm.state import DutyAccumulator, InvalidState, NetworkDuties, State +from src.constants import STAKING_MODULE_STATE_VERSION +from src.modules.oracles.staking_modules.common.state import DutyAccumulator, InvalidState, NetworkDuties, State from src.types import ValidatorIndex from src.utils.range import sequence @pytest.fixture() def state_file_path(tmp_path: Path) -> Path: - return (tmp_path / "mock").with_suffix(State.EXTENSION) + return (tmp_path / "test_oracle_cache").with_suffix(State.EXTENSION) @pytest.fixture(autouse=True) @@ -36,29 +36,29 @@ def cache_path(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: @pytest.mark.unit def test_file_returns_correct_path(self, cache_path: Path): - assert State.file() == cache_path / "cache.pkl" + assert State.file("test_oracle") == cache_path / "test_oracle_cache.pkl" @pytest.mark.unit def test_buffer_returns_correct_path(self, cache_path: Path): - state = State() - assert state.buffer == cache_path / "cache.buf" + state = State("test_oracle") + assert state.buffer == cache_path / "test_oracle_cache.buf" @pytest.mark.unit def test_load_restores_state_from_file(): - state = State() + state = State("test_oracle") state.data = { (0, 31): defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), } state.commit() - loaded_state = State.load() + loaded_state = State.load("test_oracle") assert loaded_state.data == state.data @pytest.mark.unit def test_load_returns_new_instance_if_file_not_found(state_file_path: Path): assert not state_file_path.exists() - state = State.load() + state = State.load("test_oracle") assert state.is_empty @@ -66,13 +66,13 @@ def test_load_returns_new_instance_if_file_not_found(state_file_path: Path): def test_load_returns_new_instance_if_empty_object(state_file_path: Path): with open(state_file_path, "wb") as f: pickle.dump(None, f) - state = State.load() + state = State.load("test_oracle") assert state.is_empty @pytest.mark.unit def test_commit_saves_state_to_file(state_file_path: Path, monkeypatch: pytest.MonkeyPatch): - state = State() + state = State("test_oracle") state.data = { (0, 31): defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), } @@ -88,27 +88,27 @@ def test_commit_saves_state_to_file(state_file_path: Path, monkeypatch: pytest.M @pytest.mark.unit def test_is_empty_returns_true_for_empty_state(): - state = State() + state = State("test_oracle") assert state.is_empty @pytest.mark.unit def test_is_empty_returns_false_for_non_empty_state(): - state = State() + state = State("test_oracle") state.data = {(0, 31): NetworkDuties()} assert not state.is_empty @pytest.mark.unit def test_unprocessed_epochs_raises_error_if_epochs_not_set(): - state = State() + state = State("test_oracle") with pytest.raises(ValueError, match="Epochs to process are not set"): state.unprocessed_epochs @pytest.mark.unit def test_unprocessed_epochs_returns_correct_set(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 63)) assert state.unprocessed_epochs == set(sequence(64, 95)) @@ -116,7 +116,7 @@ def test_unprocessed_epochs_returns_correct_set(): @pytest.mark.unit def test_is_fulfilled_returns_true_if_no_unprocessed_epochs(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 95)) assert state.is_fulfilled @@ -124,7 +124,7 @@ def test_is_fulfilled_returns_true_if_no_unprocessed_epochs(): @pytest.mark.unit def test_is_fulfilled_returns_false_if_unprocessed_epochs_exist(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 63)) assert not state.is_fulfilled @@ -146,7 +146,7 @@ def test_calculate_frames_raises_error_for_insufficient_epochs(): @pytest.mark.unit def test_clear_resets_state_to_empty(): - state = State() + state = State("test_oracle") state.data = {(0, 31): defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)})} state.clear() assert state.is_empty @@ -154,14 +154,14 @@ def test_clear_resets_state_to_empty(): @pytest.mark.unit def test_find_frame_returns_correct_frame(): - state = State() + state = State("test_oracle") state.data = {(0, 31): {}} assert state.find_frame(15) == (0, 31) @pytest.mark.unit def test_find_frame_raises_error_for_out_of_range_epoch(): - state = State() + state = State("test_oracle") state.data = {(0, 31): {}} with pytest.raises(ValueError, match="Epoch 32 is out of frames range"): state.find_frame(32) @@ -169,7 +169,7 @@ def test_find_frame_raises_error_for_out_of_range_epoch(): @pytest.mark.unit def test_increment_att_duty_adds_duty_correctly(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -182,7 +182,7 @@ def test_increment_att_duty_adds_duty_correctly(): @pytest.mark.unit def test_increment_prop_duty_adds_duty_correctly(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -195,7 +195,7 @@ def test_increment_prop_duty_adds_duty_correctly(): @pytest.mark.unit def test_increment_sync_duty_adds_duty_correctly(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -208,7 +208,7 @@ def test_increment_sync_duty_adds_duty_correctly(): @pytest.mark.unit def test_increment_att_duty_creates_new_validator_entry(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -221,7 +221,7 @@ def test_increment_att_duty_creates_new_validator_entry(): @pytest.mark.unit def test_increment_prop_duty_creates_new_validator_entry(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -234,7 +234,7 @@ def test_increment_prop_duty_creates_new_validator_entry(): @pytest.mark.unit def test_increment_sync_duty_creates_new_validator_entry(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -247,7 +247,7 @@ def test_increment_sync_duty_creates_new_validator_entry(): @pytest.mark.unit def test_increment_att_duty_handles_non_included_duty(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -260,7 +260,7 @@ def test_increment_att_duty_handles_non_included_duty(): @pytest.mark.unit def test_increment_prop_duty_handles_non_included_duty(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -273,7 +273,7 @@ def test_increment_prop_duty_handles_non_included_duty(): @pytest.mark.unit def test_increment_sync_duty_handles_non_included_duty(): - state = State() + state = State("test_oracle") frame = (0, 31) duty_epoch, _ = frame state.data = { @@ -286,7 +286,7 @@ def test_increment_sync_duty_handles_non_included_duty(): @pytest.mark.unit def test_increment_att_duty_raises_error_for_out_of_range_epoch(): - state = State() + state = State("test_oracle") state.att_data = { (0, 31): defaultdict(DutyAccumulator), } @@ -296,7 +296,7 @@ def test_increment_att_duty_raises_error_for_out_of_range_epoch(): @pytest.mark.unit def test_increment_prop_duty_raises_error_for_out_of_range_epoch(): - state = State() + state = State("test_oracle") state.att_data = { (0, 31): defaultdict(DutyAccumulator), } @@ -306,7 +306,7 @@ def test_increment_prop_duty_raises_error_for_out_of_range_epoch(): @pytest.mark.unit def test_increment_sync_duty_raises_error_for_out_of_range_epoch(): - state = State() + state = State("test_oracle") state.att_data = { (0, 31): defaultdict(DutyAccumulator), } @@ -316,14 +316,14 @@ def test_increment_sync_duty_raises_error_for_out_of_range_epoch(): @pytest.mark.unit def test_add_processed_epoch_adds_epoch_to_processed_set(): - state = State() + state = State("test_oracle") state.add_processed_epoch(5) assert 5 in state._processed_epochs @pytest.mark.unit def test_add_processed_epoch_does_not_duplicate_epochs(): - state = State() + state = State("test_oracle") state.add_processed_epoch(5) state.add_processed_epoch(5) assert len(state._processed_epochs) == 1 @@ -331,8 +331,8 @@ def test_add_processed_epoch_does_not_duplicate_epochs(): @pytest.mark.unit def test_migrate_discards_data_on_version_change(): - state = State() - state._version = CSM_STATE_VERSION + 1 + state = State("test_oracle") + state._version = STAKING_MODULE_STATE_VERSION + 1 state.clear = Mock() state.commit = Mock() state.migrate(0, 63, 32) @@ -341,12 +341,12 @@ def test_migrate_discards_data_on_version_change(): state.commit.assert_called_once() assert state.frames == [(0, 31), (32, 63)] assert state._epochs_to_process == tuple(sequence(0, 63)) - assert state.version == CSM_STATE_VERSION + assert state.version == STAKING_MODULE_STATE_VERSION @pytest.mark.unit def test_migrate_no_migration_needed(): - state = State() + state = State("test_oracle") state.data = { (0, 31): defaultdict(DutyAccumulator), (32, 63): defaultdict(DutyAccumulator), @@ -357,13 +357,13 @@ def test_migrate_no_migration_needed(): assert state.frames == [(0, 31), (32, 63)] assert state._epochs_to_process == tuple(sequence(0, 63)) - assert state.version == CSM_STATE_VERSION + assert state.version == STAKING_MODULE_STATE_VERSION state.commit.assert_not_called() @pytest.mark.unit def test_migrate_migrates_data(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), @@ -388,13 +388,13 @@ def test_migrate_migrates_data(): } assert state.frames == [(0, 63)] assert state._epochs_to_process == tuple(sequence(0, 63)) - assert state.version == CSM_STATE_VERSION + assert state.version == STAKING_MODULE_STATE_VERSION state.commit.assert_called_once() @pytest.mark.unit def test_migrate_invalidates_unmigrated_frames(): - state = State() + state = State("test_oracle") state._consensus_version = 1 state.data = { (0, 63): NetworkDuties( @@ -418,7 +418,7 @@ def test_migrate_invalidates_unmigrated_frames(): @pytest.mark.unit def test_migrate_discards_unmigrated_frame(): - state = State() + state = State("test_oracle") state._consensus_version = 1 state.data = { (0, 31): NetworkDuties( @@ -462,7 +462,7 @@ def test_migrate_discards_unmigrated_frame(): @pytest.mark.unit def test_migrate_frames_data_creates_new_data_correctly(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), @@ -492,7 +492,7 @@ def test_migrate_frames_data_creates_new_data_correctly(): @pytest.mark.unit def test_migrate_frames_data_handles_no_migration(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), @@ -517,7 +517,7 @@ def test_migrate_frames_data_handles_no_migration(): @pytest.mark.unit def test_migrate_frames_data_handles_partial_migration(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 5)}), @@ -552,7 +552,7 @@ def test_migrate_frames_data_handles_partial_migration(): @pytest.mark.unit def test_migrate_frames_data_handles_no_data(): - state = State() + state = State("test_oracle") state.data = {frame: NetworkDuties() for frame in state.frames} new_frames = [(0, 31)] @@ -563,7 +563,7 @@ def test_migrate_frames_data_handles_no_data(): @pytest.mark.unit def test_migrate_frames_data_handles_wider_old_frame(): - state = State() + state = State("test_oracle") state.data = { (0, 63): NetworkDuties( attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(30, 20)}), @@ -585,7 +585,7 @@ def test_migrate_frames_data_handles_wider_old_frame(): @pytest.mark.unit def test_validate_raises_error_if_state_not_fulfilled(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 94)) with pytest.raises(InvalidState, match="State is not fulfilled"): @@ -594,7 +594,7 @@ def test_validate_raises_error_if_state_not_fulfilled(): @pytest.mark.unit def test_validate_raises_error_if_processed_epoch_out_of_range(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 95)) state._processed_epochs.add(96) @@ -604,7 +604,7 @@ def test_validate_raises_error_if_processed_epoch_out_of_range(): @pytest.mark.unit def test_validate_raises_error_if_epoch_missing_in_processed_epochs(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 94)) state._processed_epochs = set(sequence(0, 94)) with pytest.raises(InvalidState, match="Epoch 95 missing in processed epochs"): @@ -613,7 +613,7 @@ def test_validate_raises_error_if_epoch_missing_in_processed_epochs(): @pytest.mark.unit def test_validate_passes_for_fulfilled_state(): - state = State() + state = State("test_oracle") state._epochs_to_process = tuple(sequence(0, 95)) state._processed_epochs = set(sequence(0, 95)) state.validate(0, 95) @@ -627,7 +627,7 @@ def test_attestation_aggregate_perf(): @pytest.mark.unit def test_get_validator_duties(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict( @@ -655,7 +655,7 @@ def test_get_validator_duties(): @pytest.mark.unit def test_get_att_network_aggr_computes_correctly(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( attestations=defaultdict( @@ -671,7 +671,7 @@ def test_get_att_network_aggr_computes_correctly(): @pytest.mark.unit def test_get_sync_network_aggr_computes_correctly(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( syncs=defaultdict( @@ -687,7 +687,7 @@ def test_get_sync_network_aggr_computes_correctly(): @pytest.mark.unit def test_get_prop_network_aggr_computes_correctly(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties( proposals=defaultdict( @@ -703,7 +703,7 @@ def test_get_prop_network_aggr_computes_correctly(): @pytest.mark.unit def test_get_att_network_aggr_raises_error_for_invalid_accumulator(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties(attestations=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 15)})) } @@ -713,7 +713,7 @@ def test_get_att_network_aggr_raises_error_for_invalid_accumulator(): @pytest.mark.unit def test_get_prop_network_aggr_raises_error_for_invalid_accumulator(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties(proposals=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 15)})) } @@ -723,7 +723,7 @@ def test_get_prop_network_aggr_raises_error_for_invalid_accumulator(): @pytest.mark.unit def test_get_sync_network_aggr_raises_error_for_invalid_accumulator(): - state = State() + state = State("test_oracle") state.data = { (0, 31): NetworkDuties(syncs=defaultdict(DutyAccumulator, {ValidatorIndex(1): DutyAccumulator(10, 15)})) } @@ -733,28 +733,28 @@ def test_get_sync_network_aggr_raises_error_for_invalid_accumulator(): @pytest.mark.unit def test_get_att_network_aggr_raises_error_for_missing_frame_data(): - state = State() + state = State("test_oracle") with pytest.raises(ValueError, match="No data for frame"): state.get_att_network_aggr((0, 31)) @pytest.mark.unit def test_get_prop_network_aggr_raises_error_for_missing_frame_data(): - state = State() + state = State("test_oracle") with pytest.raises(ValueError, match="No data for frame"): state.get_prop_network_aggr((0, 31)) @pytest.mark.unit def test_get_sync_network_aggr_raises_error_for_missing_frame_data(): - state = State() + state = State("test_oracle") with pytest.raises(ValueError, match="No data for frame"): state.get_sync_network_aggr((0, 31)) @pytest.mark.unit def test_get_att_network_aggr_handles_empty_frame_data(): - state = State() + state = State("test_oracle") state.data = {(0, 31): NetworkDuties()} aggr = state.get_att_network_aggr((0, 31)) assert aggr.assigned == 0 @@ -763,7 +763,7 @@ def test_get_att_network_aggr_handles_empty_frame_data(): @pytest.mark.unit def test_get_prop_network_aggr_handles_empty_frame_data(): - state = State() + state = State("test_oracle") state.data = {(0, 31): NetworkDuties()} aggr = state.get_prop_network_aggr((0, 31)) assert aggr.assigned == 0 @@ -772,7 +772,7 @@ def test_get_prop_network_aggr_handles_empty_frame_data(): @pytest.mark.unit def test_get_sync_network_aggr_handles_empty_frame_data(): - state = State() + state = State("test_oracle") state.data = {(0, 31): NetworkDuties()} aggr = state.get_sync_network_aggr((0, 31)) assert aggr.assigned == 0 diff --git a/tests/modules/csm/test_strikes.py b/tests/modules/csm/test_strikes.py index 1ec3e7c66..075c962ad 100644 --- a/tests/modules/csm/test_strikes.py +++ b/tests/modules/csm/test_strikes.py @@ -1,7 +1,7 @@ import pytest from eth_utils.types import is_list_like -from src.modules.csm.types import StrikesList +from src.modules.oracles.staking_modules.common.types import StrikesList @pytest.mark.unit diff --git a/tests/modules/csm/test_tree.py b/tests/modules/csm/test_tree.py index 2ce44aca0..f34692703 100644 --- a/tests/modules/csm/test_tree.py +++ b/tests/modules/csm/test_tree.py @@ -6,8 +6,8 @@ from hexbytes import HexBytes from src.constants import UINT64_MAX -from src.modules.csm.tree import RewardsTree, StandardMerkleTree, StrikesTree, Tree -from src.modules.csm.types import RewardsTreeLeaf, StrikesList, StrikesTreeLeaf +from src.modules.oracles.staking_modules.common.tree import RewardsTree, StandardMerkleTree, StrikesTree, Tree +from src.modules.oracles.staking_modules.common.types import RewardsTreeLeaf, StrikesList, StrikesTreeLeaf from src.types import NodeOperatorId from src.utils.types import hex_str_to_bytes diff --git a/tests/modules/ejector/test_data_encode.py b/tests/modules/ejector/test_data_encode.py index bf368dcca..a62b385e7 100644 --- a/tests/modules/ejector/test_data_encode.py +++ b/tests/modules/ejector/test_data_encode.py @@ -4,7 +4,7 @@ import pytest -from src.modules.ejector.data_encode import ( +from src.modules.oracles.ejector.data_encode import ( MODULE_ID_LENGTH, NODE_OPERATOR_ID_LENGTH, VALIDATOR_INDEX_LENGTH, diff --git a/tests/modules/ejector/test_ejector.py b/tests/modules/ejector/test_ejector.py index 32681ac4f..7b26bc0f1 100644 --- a/tests/modules/ejector/test_ejector.py +++ b/tests/modules/ejector/test_ejector.py @@ -14,11 +14,11 @@ MIN_ACTIVATION_BALANCE, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) -from src.modules.ejector import ejector as ejector_module -from src.modules.ejector.ejector import Ejector, logger as ejector_logger -from src.modules.ejector.types import EjectorProcessingState -from src.modules.submodules.oracle_module import ModuleExecuteDelay -from src.modules.submodules.types import ChainConfig, CurrentFrame +from src.modules.oracles.ejector import ejector as ejector_module +from src.modules.oracles.ejector.ejector import Ejector, logger as ejector_logger +from src.modules.oracles.ejector.types import EjectorProcessingState +from src.modules.oracles.common.oracle_module import ModuleExecuteDelay +from src.modules.common.types import ChainConfig, CurrentFrame from src.providers.consensus.types import ( BeaconStateView, ) diff --git a/tests/modules/ejector/test_prediction.py b/tests/modules/ejector/test_prediction.py index d494bb803..e425be45f 100644 --- a/tests/modules/ejector/test_prediction.py +++ b/tests/modules/ejector/test_prediction.py @@ -5,7 +5,7 @@ from web3.types import Wei import src.services.prediction as prediction_module -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.services.prediction import RewardsPredictionService from src.types import BlockNumber, SlotNumber from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/ejector/test_sweep.py b/tests/modules/ejector/test_sweep.py index 2461469ab..f98ae742f 100644 --- a/tests/modules/ejector/test_sweep.py +++ b/tests/modules/ejector/test_sweep.py @@ -3,15 +3,15 @@ import pytest -import src.modules.ejector.sweep as sweep_module +import src.modules.oracles.ejector.sweep as sweep_module from src.constants import MAX_WITHDRAWALS_PER_PAYLOAD, MIN_ACTIVATION_BALANCE -from src.modules.ejector.sweep import ( +from src.modules.oracles.ejector.sweep import ( get_pending_partial_withdrawals, get_sweep_delay_in_epochs, get_validators_withdrawals, Withdrawal, ) -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.providers.consensus.types import BeaconStateView, PendingPartialWithdrawal from src.types import Gwei from tests.factory.consensus import BeaconStateViewFactory diff --git a/tests/modules/ejector/test_validator_exit_order_iterator.py b/tests/modules/ejector/test_validator_exit_order_iterator.py index 1e4044ef5..3637e3414 100644 --- a/tests/modules/ejector/test_validator_exit_order_iterator.py +++ b/tests/modules/ejector/test_validator_exit_order_iterator.py @@ -2,7 +2,7 @@ import pytest -from src.modules.submodules.types import ChainConfig +from src.modules.common.types import ChainConfig from src.services.exit_order_iterator import NodeOperatorStats, StakingModuleStats, ValidatorExitIterator from src.web3py.extensions.lido_validators import NodeOperatorLimitMode from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/performance_collector/test_checkpoint.py b/tests/modules/performance_collector/test_checkpoint.py index a898cc862..6411aa7c2 100644 --- a/tests/modules/performance_collector/test_checkpoint.py +++ b/tests/modules/performance_collector/test_checkpoint.py @@ -4,9 +4,9 @@ import pytest -import src.modules.performance.collector.checkpoint as checkpoint_module +import src.modules.sidecars.performance.collector.checkpoint as checkpoint_module from src.constants import EPOCHS_PER_SYNC_COMMITTEE_PERIOD -from src.modules.performance.collector.checkpoint import ( +from src.modules.sidecars.performance.collector.checkpoint import ( FrameCheckpoint, FrameCheckpointProcessor, FrameCheckpointsIterator, @@ -15,9 +15,9 @@ SyncCommitteesCache, process_attestations, ) -from src.modules.performance.common.db import DutiesDB -from src.modules.performance.common.types import AttDutyMisses, ProposalDuty, SyncDuty -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.sidecars.performance.common.db import DutiesDB +from src.modules.sidecars.performance.common.types import AttDutyMisses, ProposalDuty, SyncDuty +from src.modules.common.types import ChainConfig, FrameConfig from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import BeaconSpecResponse, BlockAttestation, SlotAttestationCommittee, SyncCommittee from src.types import BlockRoot, EpochNumber, ValidatorIndex @@ -60,7 +60,9 @@ def converter(frame_config: FrameConfig, chain_config: ChainConfig) -> Web3Conve @pytest.fixture def sync_committees_cache(): - with patch('src.modules.performance.collector.checkpoint.SYNC_COMMITTEES_CACHE', SyncCommitteesCache()) as cache: + with patch( + 'src.modules.sidecars.performance.collector.checkpoint.SYNC_COMMITTEES_CACHE', SyncCommitteesCache() + ) as cache: yield cache @@ -446,7 +448,8 @@ def test_get_sync_committee_fetches_and_caches_when_not_cached( prev_slot_response.message.slot = SlotNumber(0) prev_slot_response.message.body.execution_payload.block_hash = "0x00" with patch( - 'src.modules.performance.collector.checkpoint.get_prev_non_missed_slot', Mock(return_value=prev_slot_response) + 'src.modules.sidecars.performance.collector.checkpoint.get_prev_non_missed_slot', + Mock(return_value=prev_slot_response), ): result = frame_checkpoint_processor._get_sync_committee(epoch) @@ -472,7 +475,8 @@ def test_get_sync_committee_handles_cache_eviction( prev_slot_response.message.slot = SlotNumber(0) prev_slot_response.message.body.execution_payload.block_hash = "0x00" with patch( - 'src.modules.performance.collector.checkpoint.get_prev_non_missed_slot', Mock(return_value=prev_slot_response) + 'src.modules.sidecars.performance.collector.checkpoint.get_prev_non_missed_slot', + Mock(return_value=prev_slot_response), ): result = frame_checkpoint_processor._get_sync_committee(epoch) @@ -530,7 +534,8 @@ def test_get_dependent_root_for_proposer_duties_from_cl_when_slot_out_of_range(f prev_slot_response = Mock() prev_slot_response.message.slot = non_missed_slot with patch( - 'src.modules.performance.collector.checkpoint.get_prev_non_missed_slot', Mock(return_value=prev_slot_response) + 'src.modules.sidecars.performance.collector.checkpoint.get_prev_non_missed_slot', + Mock(return_value=prev_slot_response), ): frame_checkpoint_processor.cc.get_block_root = Mock(return_value=Mock(root=checkpoint_block_roots[0])) diff --git a/tests/modules/performance_collector/test_performance_collector.py b/tests/modules/performance_collector/test_performance_collector.py index 7f3cbc29c..ee74dc9e8 100644 --- a/tests/modules/performance_collector/test_performance_collector.py +++ b/tests/modules/performance_collector/test_performance_collector.py @@ -1,8 +1,8 @@ import pytest from unittest.mock import Mock, patch -from src.modules.performance.collector.collector import PerformanceCollector -from src.modules.performance.common.db import DutiesDB, EpochsDemand +from src.modules.sidecars.performance.collector.collector import PerformanceCollector +from src.modules.sidecars.performance.common.db import DutiesDB, EpochsDemand from src.types import EpochNumber @@ -21,9 +21,9 @@ def mock_db(): @pytest.fixture def performance_collector(mock_w3, mock_db): """Create PerformanceCollector instance with mocked dependencies""" - with patch('src.modules.performance.collector.collector.DutiesDB', return_value=mock_db), patch( - 'src.modules.performance.web.server.serve' - ), patch('src.modules.performance.web.server.PERFORMANCE_WEB_SERVER_API_PORT', 8080): + with patch('src.modules.sidecars.performance.collector.collector.DutiesDB', return_value=mock_db), patch( + 'src.modules.sidecars.performance.web.server.serve' + ), patch('src.modules.sidecars.performance.web.server.PERFORMANCE_WEB_SERVER_API_PORT', 8080): mock_db.get_epochs_demands_max_updated_at.return_value = 0 collector = PerformanceCollector(mock_w3) return collector diff --git a/tests/modules/performance_collector/test_processing_attestation.py b/tests/modules/performance_collector/test_processing_attestation.py index 785631844..78ba8fccd 100644 --- a/tests/modules/performance_collector/test_processing_attestation.py +++ b/tests/modules/performance_collector/test_processing_attestation.py @@ -3,7 +3,7 @@ import pytest -from src.modules.performance.collector.checkpoint import ( +from src.modules.sidecars.performance.collector.checkpoint import ( get_committee_indices, hex_bitlist_to_list, hex_bitvector_to_list, diff --git a/tests/modules/submodules/consensus/conftest.py b/tests/modules/submodules/consensus/conftest.py index a27bb1ffc..e95e1b0fe 100644 --- a/tests/modules/submodules/consensus/conftest.py +++ b/tests/modules/submodules/consensus/conftest.py @@ -1,6 +1,6 @@ import pytest -from src.modules.submodules.consensus import ConsensusModule +from src.modules.oracles.common.consensus import ConsensusModule from src.types import BlockStamp, ReferenceBlockStamp diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 1ccdee3b4..e3bb24dd8 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -8,10 +8,10 @@ from web3.exceptions import ContractCustomError from src import variables -from src.modules.submodules import consensus as consensus_module -from src.modules.submodules.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo -from src.modules.submodules.exceptions import IncompatibleOracleVersion, ContractVersionMismatch -from src.modules.submodules.types import ChainConfig +from src.modules.oracles.common import consensus as consensus_module +from src.modules.oracles.common.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo +from src.modules.oracles.common.exceptions import IncompatibleOracleVersion, ContractVersionMismatch +from src.modules.common.types import ChainConfig from src.providers.consensus.types import BeaconSpecResponse from src.types import BlockStamp, ReferenceBlockStamp diff --git a/tests/modules/submodules/consensus/test_reports.py b/tests/modules/submodules/consensus/test_reports.py index 93927b69c..f971d7bab 100644 --- a/tests/modules/submodules/consensus/test_reports.py +++ b/tests/modules/submodules/consensus/test_reports.py @@ -7,9 +7,9 @@ from dataclasses import dataclass from src import variables -from src.modules.accounting.accounting import Accounting -from src.modules.accounting.types import ReportData -from src.modules.submodules.types import ChainConfig, FrameConfig, ZERO_HASH +from src.modules.oracles.accounting.accounting import Accounting +from src.modules.oracles.accounting.types import ReportData +from src.modules.common.types import ChainConfig, FrameConfig, ZERO_HASH from src.types import SlotNumber, Gwei, StakingModuleId from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/submodules/test_oracle_module.py b/tests/modules/submodules/test_oracle_module.py index a7bd1eb6a..4989e3722 100644 --- a/tests/modules/submodules/test_oracle_module.py +++ b/tests/modules/submodules/test_oracle_module.py @@ -9,11 +9,11 @@ from web3_multi_provider.multi_http_provider import NoActiveProviderError from src import variables -from src.modules.submodules.exceptions import ( +from src.modules.oracles.common.exceptions import ( IncompatibleOracleVersion, IsNotMemberException, ) -from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay +from src.modules.oracles.common.oracle_module import BaseModule, ModuleExecuteDelay from src.providers.http_provider import NotOkResponse from src.providers.keys.client import KeysOutdatedException from src.types import BlockStamp @@ -32,6 +32,9 @@ def execute_module(self, blockstamp): def refresh_contracts(self): pass + def is_contracts_addresses_changed(self): + pass + @pytest.fixture(autouse=True) def set_default_sleep(monkeypatch): @@ -65,22 +68,22 @@ def test_receive_last_finalized_slot(oracle): @pytest.mark.unit @responses.activate def test_cycle_handler_run_once_per_slot(oracle, web3): - web3.lido_contracts.has_contract_address_changed = Mock() + oracle.is_contracts_addresses_changed = Mock() oracle._receive_last_finalized_slot = Mock(return_value=ReferenceBlockStampFactory.build(slot_number=1)) responses.get('http://localhost:8000/pulse/', status=HTTPStatus.OK) oracle.cycle_handler() assert oracle.call_count == 1 - assert web3.lido_contracts.has_contract_address_changed.call_count == 1 + assert oracle.is_contracts_addresses_changed.call_count == 1 oracle.cycle_handler() assert oracle.call_count == 1 - assert web3.lido_contracts.has_contract_address_changed.call_count == 1 + assert oracle.is_contracts_addresses_changed.call_count == 1 oracle._receive_last_finalized_slot = Mock(return_value=ReferenceBlockStampFactory.build(slot_number=2)) oracle.cycle_handler() assert oracle.call_count == 2 - assert web3.lido_contracts.has_contract_address_changed.call_count == 2 + assert oracle.is_contracts_addresses_changed.call_count == 2 @pytest.mark.unit diff --git a/tests/providers_clients/test_keys_api_client.py b/tests/providers_clients/test_keys_api_client.py index 954384e49..275bdb86a 100644 --- a/tests/providers_clients/test_keys_api_client.py +++ b/tests/providers_clients/test_keys_api_client.py @@ -80,11 +80,11 @@ def test_get_used_module_operators_keys__csm_module__response_data_is_valid( empty_blockstamp, ): csm_module_operators_keys = keys_api_client.get_used_module_operators_keys( - module_address=variables.CSM_MODULE_ADDRESS, # type: ignore + module_address=variables.CS_MODULE_ADDRESS, # type: ignore blockstamp=empty_blockstamp, ) - assert csm_module_operators_keys['module']['stakingModuleAddress'] == variables.CSM_MODULE_ADDRESS + assert csm_module_operators_keys['module']['stakingModuleAddress'] == variables.CS_MODULE_ADDRESS assert csm_module_operators_keys['module']['id'] >= 0 assert len(csm_module_operators_keys['keys']) > 0 assert len(csm_module_operators_keys['operators']) > 0 @@ -97,7 +97,7 @@ def test_get_used_module_operators_keys__csm_module__response_data_is_valid( for operator in csm_module_operators_keys['operators']: assert operator['index'] >= 0 assert Web3.is_address(operator['rewardAddress']) - assert operator['moduleAddress'] == variables.CSM_MODULE_ADDRESS + assert operator['moduleAddress'] == variables.CS_MODULE_ADDRESS def test_get_status__response_version_is_allowed( self, diff --git a/tests/utils/test_apr.py b/tests/utils/test_apr.py index dab9d6f30..38fa2f387 100644 --- a/tests/utils/test_apr.py +++ b/tests/utils/test_apr.py @@ -2,7 +2,7 @@ import pytest -from src.modules.accounting.types import SECONDS_IN_YEAR +from src.modules.oracles.accounting.types import SECONDS_IN_YEAR from src.utils.apr import calculate_gross_core_apr pytestmark = pytest.mark.unit @@ -88,7 +88,7 @@ def test_negative_growth_apr(self): def test_large_numbers_precision(self): """Test APR calculation with high precision values.""" apr = calculate_gross_core_apr( - post_internal_ether=10**27 + 5 * 10**25, # increased на 5% + post_internal_ether=10**27 + 5 * 10**25, # increased by 5% post_internal_shares=10**27, shares_minted_as_fees=0, pre_total_ether=10**27, diff --git a/tests/utils/test_web3_converter.py b/tests/utils/test_web3_converter.py index b091dfae3..bd37fcf6d 100644 --- a/tests/utils/test_web3_converter.py +++ b/tests/utils/test_web3_converter.py @@ -1,6 +1,6 @@ import pytest -from src.modules.submodules.types import ChainConfig, FrameConfig +from src.modules.common.types import ChainConfig, FrameConfig from src.types import EpochNumber, FrameNumber, SlotNumber from src.utils.web3converter import Web3Converter from tests.factory.configs import ChainConfigFactory, FrameConfigFactory diff --git a/tests/web3py/test_lido_validators.py b/tests/web3py/test_lido_validators.py index 2d1319786..6a149b203 100644 --- a/tests/web3py/test_lido_validators.py +++ b/tests/web3py/test_lido_validators.py @@ -2,7 +2,7 @@ import pytest -from src.modules.accounting.types import BeaconStat +from src.modules.oracles.accounting.types import BeaconStat from src.web3py.extensions.lido_validators import CountOfKeysDiffersException from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.no_registry import (