feed/docker-compose.yml at main · hathitrust/feed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
---

x-condition-healthy: &healthy
  condition: service_healthy

x-healthcheck-defaults: &healthcheck-defaults
  interval: 5s
  timeout: 10s
  start_period: 10s
  retries: 5

services:

  test:
    build: .
    volumes:
      - .:/usr/local/feed
      - ./clamav:/var/lib/clamav
    environment:
      - HTFEED_CONFIG=/usr/local/feed/etc/config_test.yml
      - FEED_HOME=/usr/local/feed
      - VERSION=feed-development
      - PUSHGATEWAY=http://pushgateway:9091
      - TEST=1
    command: prove
    depends_on:
      mariadb: *healthy
      pushgateway: *healthy
      rabbitmq: *healthy
      versitygw: *healthy

  # Note: for permissions purposes this does NOT bind in the local development
  # environment, so local changes after running docker compose build will NOT
  # be reflected.
  test-and-cover:
    build: .
    volumes:
      - ./clamav:/var/lib/clamav
      - ./var/vgw:/usr/local/feed/var/vgw
    environment:
      - HTFEED_CONFIG=/usr/local/feed/etc/config_test.yml
      - FEED_HOME=/usr/local/feed
      - VERSION=feed-development
      - PUSHGATEWAY=http://pushgateway:9091
      - TEST=1
      # pass through info needed by coveralls uploader
      - GITHUB_TOKEN
      - GITHUB_RUN_ID
      - GITHUB_EVENT_TOKEN
      - GITHUB_EVENT_PATH
      - GITHUB_SHA
      - GITHUB_REF
      - GITHUB_ACTIONS
    command: cover -test -report Coveralls -make 'prove; exit $?'
    depends_on:
      mariadb: *healthy
      pushgateway: *healthy
      rabbitmq: *healthy
      versitygw: *healthy

  ingest:
    build: .
    volumes:
      - .:/usr/local/feed
      - ./ingest_sips:/tmp/sips
      - ./ingest_stage:/tmp/stage
      - ~/.config/rclone/rclone.conf:/usr/local/feed/etc/rclone.conf
      - ./clamav:/var/lib/clamav
      # FIXME -- before these will work, run: docker compose run --rm ingest chown ingest.ingest /sdr1 /sdr2 /htdataden
      - repository_link:/sdr1
      - repository_obj:/sdr2
      - backups:/htdataden
    command:
      - /usr/bin/perl
      - /usr/local/feed/bin/feedd.pl
    environment:
      - HTFEED_CONFIG=/usr/local/feed/etc/config_ingest_docker.yml
      - FEED_HOME=/usr/local/feed
      - VERSION=feed-development
      - HTFEED_JOBMETRICS_DATA_DIR=/usr/local/feed/jobmetrics
    depends_on:
      mariadb: *healthy

  validate:
    build: .
    volumes:
      - .:/usr/local/feed
      - ./volumes_to_test:/tmp/stage/toingest/test
    environment:
      - HTFEED_CONFIG=/usr/local/feed/etc/config_prevalidate.yml
      - HTFEED_JOBMETRICS_DATA_DIR=/usr/local/feed/jobmetrics
      - FEED_HOME=/usr/local/feed
      - VERSION=feed-development
    command: bash -c "/bin/ls /tmp/stage/toingest/test/*.zip | xargs -n 1 basename | sed s/.zip// | perl -w /usr/local/feed/bin/validate_volume.pl -p simple -n test --no-clean"
    depends_on:
      mariadb: *healthy

  prometheus-exporter:
    build: .
    volumes:
      - .:/usr/local/feed
    ports:
      - "9090:9090"
    environment:
      - HTFEED_CONFIG=/usr/local/feed/etc/config_prevalidate.yml
      - HTFEED_JOBMETRICS_DATA_DIR=/usr/local/feed/jobmetrics
      - FEED_HOME=/usr/local/feed
      - VERSION=feed-development
    command: plackup -p 9090 ./bin/metrics_exporter.pl


  mariadb:
    image: ghcr.io/hathitrust/db-image:latest
    volumes:
      - ./etc/ingest.sql:/docker-entrypoint-initdb.d/999-ingest.sql
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: mysqlroot
    healthcheck:
      <<: *healthcheck-defaults
      test: ["CMD", "healthcheck.sh", "--su-mysql", "--connect", "--innodb_initialized"]

  # S3 -> filesystem gateway
  versitygw:
    user: "1000:1000"
    image: versity/versitygw
    restart: always
    environment:
      ROOT_ACCESS_KEY: TESTINGACCESSKEY
      ROOT_SECRET_KEY: testingsecretkey
    volumes:
      - ./var/vgw:/usr/local/feed/var/vgw
      - ./var/metadata_vgw:/usr/local/feed/var/metadata_vgw
    command: --health /health posix --sidecar /usr/local/feed/var/metadata_vgw /usr/local/feed/var/vgw
    healthcheck:
      <<: *healthcheck-defaults
      test: [ "CMD", "wget", "--quiet", "--tries=1", "-O", "/dev/null", "http://127.0.0.1:7070/health" ]

  pushgateway:
    image: prom/pushgateway
    command:
      - --web.enable-admin-api
    ports:
      - "9091:9091"
    healthcheck:
      <<: *healthcheck-defaults
      test: [ "CMD", "wget", "--quiet", "--tries=1", "-O", "/dev/null", "http://127.0.0.1:9091/-/healthy" ]


  rabbitmq:
    image: rabbitmq
    healthcheck:
      <<: *healthcheck-defaults
      test: ["CMD", "rabbitmqctl", "status"]

volumes:
  repository_link:
  repository_obj:
  backups:
  vgw_sidecar: