Skip to content

Commit b701be4

Browse files
nicoalencarnicoalencar
authored
Merge pull request #153 from 0xProject/nicolas/deng-361-fix-and-improve-local-airflow-env
Moving db related configs to 0x-migrations
2 parents 563cb6d + 101aa0c commit b701be4

File tree

5 files changed

+17
-69
lines changed

5 files changed

+17
-69
lines changed

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ Configure the scape **Mode**
1414

1515
2. Start Postgres
1616

17-
```sh
18-
docker-compose up -d # get postgres up
19-
```
17+
For local development, you will need to get the local databases running depending on which pipelines you are going to develop/test.
18+
You can setup them by following the instructions on the [0x-data-migrations](https://github.com/0xProject/0x-data-migrations) repository. We manage all db setup/changes there.
19+
It will guide you on how to get the dbs up and running the initial setup that is necessary to be used here.
2020

2121
3. Build the Scraper images
2222

‎build_local.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env sh
22

3-
docker-compose -f docker-compose-dev.yml up -d postgres
43
docker-compose -f docker-compose-dev.yml build event-pipeline-ethereum
54
docker-compose -f docker-compose-dev.yml build event-pipeline-base
65

‎docker-compose-dev.yml

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,5 @@
11
version: '3'
22
services:
3-
postgres:
4-
image: postgres:13.4
5-
environment:
6-
POSTGRES_USER: '${POSTGRES_USER}'
7-
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
8-
POSTGRES_DB: '${POSTGRES_DB}'
9-
POSTGRES_PORT: ${POSTGRES_PORT}
10-
# persist the postgres data to disk so we don't lose it
11-
# on rebuilds.
12-
volumes:
13-
- ./mounts/postgres:/var/lib/postgresql/data
14-
- ./postgres-init.sql:/docker-entrypoint-initdb.d/postgres-init.sql
15-
ports:
16-
- '5432:5432'
17-
command: ["postgres", "-c", "log_statement=none", "-c", "log_destination=stderr"]
183

194
event-pipeline-ethereum:
205
depends_on:
@@ -166,7 +151,6 @@ services:
166151
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
167152
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7'
168153

169-
170154
event-pipeline-fantom:
171155
depends_on:
172156
- postgres

‎docker-compose.yml

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,14 @@
11
version: '3'
22
services:
3-
postgres:
4-
image: postgres:13.4
5-
environment:
6-
POSTGRES_USER: '${POSTGRES_USER}'
7-
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
8-
POSTGRES_DB: '${POSTGRES_DB}'
9-
POSTGRES_PORT: ${POSTGRES_PORT}
10-
# persist the postgres data to disk so we don't lose it
11-
# on rebuilds.
12-
volumes:
13-
- ./mounts/postgres:/var/lib/postgresql/data
14-
ports:
15-
- '5432:5432'
16-
command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"]
17-
183
event-pipeline-ethereum:
19-
depends_on:
20-
- postgres
214
build:
225
context: .
236
dockerfile: Dockerfile
247
restart: always
258
environment:
269
EVM_RPC_URL: '${RPC_URL_ETHEREUM}'
2710
CHAIN_ID: '1'
28-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
11+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
2912
SCHEMA: 'events'
3013
# FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true"
3114
# TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266
@@ -85,16 +68,14 @@ services:
8568
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
8669

8770
event-pipeline-bsc:
88-
depends_on:
89-
- postgres
9071
build:
9172
context: .
9273
dockerfile: Dockerfile
9374
restart: always
9475
environment:
9576
EVM_RPC_URL: '${RPC_URL_BSC}'
9677
CHAIN_ID: '56'
97-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
78+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
9879
SCHEMA: 'events_bsc'
9980
SCRAPER_MODE: 'BLOCKS'
10081
MAX_BLOCKS_REORG: 300
@@ -123,16 +104,14 @@ services:
123104
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
124105

125106
event-pipeline-polygon:
126-
depends_on:
127-
- postgres
128107
build:
129108
context: .
130109
dockerfile: Dockerfile
131110
restart: always
132111
environment:
133112
EVM_RPC_URL: '${RPC_URL_POLYGON}'
134113
CHAIN_ID: '137'
135-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
114+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
136115
SCHEMA: 'events_polygon'
137116
SCRAPER_MODE: 'BLOCKS'
138117
ENABLE_PROMETHEUS_METRICS: "true"
@@ -159,16 +138,14 @@ services:
159138
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
160139

161140
event-pipeline-avalanche:
162-
depends_on:
163-
- postgres
164141
build:
165142
context: .
166143
dockerfile: Dockerfile
167144
restart: always
168145
environment:
169146
EVM_RPC_URL: '${RPC_URL_AVALANCHE}'
170147
CHAIN_ID: '43114'
171-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
148+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
172149
SCHEMA: 'events_avalanche'
173150
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
174151
#KAFKA_SSL: '${KAFKA_SSL}'
@@ -196,16 +173,14 @@ services:
196173

197174

198175
event-pipeline-fantom:
199-
depends_on:
200-
- postgres
201176
build:
202177
context: .
203178
dockerfile: Dockerfile
204179
restart: always
205180
environment:
206181
EVM_RPC_URL: '${RPC_URL_FANTOM}'
207182
CHAIN_ID: '250'
208-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
183+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
209184
SCHEMA: 'events_fantom'
210185
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
211186
#KAFKA_SSL: '${KAFKA_SSL}'
@@ -228,16 +203,14 @@ services:
228203
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83'
229204

230205
event-pipeline-celo:
231-
depends_on:
232-
- postgres
233206
build:
234207
context: .
235208
dockerfile: Dockerfile
236209
restart: always
237210
environment:
238211
EVM_RPC_URL: '${RPC_URL_CELO}'
239212
CHAIN_ID: '42220'
240-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
213+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
241214
SCHEMA: 'events_celo'
242215
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
243216
#KAFKA_SSL: '${KAFKA_SSL}'
@@ -254,16 +227,14 @@ services:
254227
NFT_FEATURE_START_BLOCK: 11820000
255228

256229
event-pipeline-optimism:
257-
depends_on:
258-
- postgres
259230
build:
260231
context: .
261232
dockerfile: Dockerfile
262233
restart: always
263234
environment:
264235
EVM_RPC_URL: '${RPC_URL_OPTIMISM}'
265236
CHAIN_ID: '10'
266-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
237+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
267238
SCHEMA: 'events_optimism'
268239
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
269240
#KAFKA_SSL: '${KAFKA_SSL}'
@@ -289,16 +260,14 @@ services:
289260
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
290261

291262
event-pipeline-arbitrum:
292-
depends_on:
293-
- postgres
294263
build:
295264
context: .
296265
dockerfile: Dockerfile
297266
restart: always
298267
environment:
299268
EVM_RPC_URL: '${EVM_RPC_URL_ARBITRUM}'
300269
CHAIN_ID: '42161'
301-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
270+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
302271
SCHEMA: 'events_arbitrum'
303272
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
304273
#KAFKA_SSL: '${KAFKA_SSL}'
@@ -320,16 +289,14 @@ services:
320289
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
321290

322291
event-pipeline-base:
323-
depends_on:
324-
- postgres
325292
build:
326293
context: .
327294
dockerfile: Dockerfile
328295
restart: always
329296
environment:
330297
EVM_RPC_URL: '${RPC_URL_BASE}'
331298
CHAIN_ID: '8453'
332-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
299+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
333300
SCHEMA: 'events_base'
334301
SCRAPER_MODE: 'BLOCKS'
335302
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
@@ -352,16 +319,14 @@ services:
352319
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
353320

354321
token-scraper-ethereum:
355-
depends_on:
356-
- postgres
357322
build:
358323
context: .
359324
dockerfile: Dockerfile
360325
restart: always
361326
environment:
362327
EVM_RPC_URL: '${RPC_URL_ETHEREUM}'
363328
CHAIN_ID: '1'
364-
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
329+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
365330
SCHEMA: 'events'
366331
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
367332
#KAFKA_SSL: '${KAFKA_SSL}'

‎.env.example renamed to ‎env.tmpl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
POSTGRES_USER=scraper
2-
POSTGRES_PASSWORD=scraper
3-
POSTGRES_DB=events
1+
POSTGRES_HOST=172.17.0.1
2+
POSTGRES_USER=zeroex
3+
POSTGRES_PASSWORD=zeroex
4+
POSTGRES_DB=hashalytics
45
POSTGRES_PORT=5432
56

67
RPC_URL_ETHEREUM=
@@ -11,4 +12,3 @@ RPC_URL_FANTOM=
1112
RPC_URL_CELO=
1213
RPC_URL_OPTIMISM=
1314
RPC_URL_ARBITRUM=
14-

0 commit comments

Comments
 (0)