Merge pull request #1178 from cmu-delphi/release/delphi-epidata-4.1.0

* API keys (#1115)

* chore: release delphi-epidata 4.1.0

---------

Co-authored-by: Katie Mazaitis <[email protected]>
Co-authored-by: Dmytro Trotsko <[email protected]>
Co-authored-by: Brian Clark <[email protected]>
Co-authored-by: george haff <[email protected]>

Author: 4 people

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.12
+current_version = 4.1.0
 commit = False
 tag = False
 

.dockerignore

@@ -1,6 +1,7 @@
 /delphi-epidata
-/.mypy_cache
+**/.mypy_cache
 /.github
 /docs
-__pycache__
-/node_modules
+**/__pycache__
+**/.pytest_cache
+**/node_modules

.env.example

@@ -1,18 +1,6 @@
 FLASK_DEBUG=True
 SQLALCHEMY_DATABASE_URI=sqlite:///test.db
 FLASK_SECRET=abc
-SECRET_TWITTER=abc
-SECRET_GHT=abc
-SECRET_FLUVIEW=abc
-SECRET_CDC=abc
-SECRET_SENSORS=abc
-SECRET_SENSOR_TWTR=abc
-SECRET_SENSOR_GFT=abc
-SECRET_SENSOR_GHT=abc
-SECRET_SENSOR_GHTJ=abc
-SECRET_SENSOR_CDC=abc
-SECRET_SENSOR_QUID=abc
-SECRET_SENSOR_WIKI=abc
-SECRET_QUIDEL=abc
-SECRET_NOROSTAT=abc
-SECRET_AFHSB=abc
+#API_KEY_REQUIRED_STARTING_AT=2021-07-30
+API_KEY_ADMIN_PASSWORD=abc
+API_KEY_REGISTER_WEBHOOK_TOKEN=abc

.github/workflows/ci.yaml

@@ -53,24 +53,31 @@ jobs:
         run: |
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
           cd ./repos/delphi/delphi-epidata
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
       # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
       # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
-      - name: Start services
+      - name: Start database and Redis services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
-          docker ps
+          docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
+          
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
           chmod +x wait-for
           ./wait-for localhost:13306 -- echo 'ready'
           sleep 10s
+
+      - name: Start delphi_web_epidata
+        run: |
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker ps
+
       - name: Run Unit Tests
         run: |
           docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
@@ -81,7 +88,7 @@ jobs:
 
       - name: Clean Up
         run: |
-          docker stop delphi_database_epidata delphi_web_epidata
+          docker stop delphi_database_epidata delphi_web_epidata delphi_redis
           docker network remove delphi-net
 
   build_js_client:
@@ -108,7 +115,8 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    # TODO: #1112 Remove `|| github.ref == 'refs/heads/api-keys'` after transition to production status.
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/api-keys'
 
     runs-on: ubuntu-latest
     steps:

dev/docker/python/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.8-buster
 
+RUN apt-get update && apt-get install -y r-base && Rscript -e "install.packages(c('httr','xml2'))"
+
 WORKDIR /usr/src/app
 
 COPY repos repos

dev/docker/redis/Dockerfile

@@ -0,0 +1,5 @@
+FROM redis
+
+ENV REDIS_PASSWORD=$REDIS_PASSWORD
+
+CMD ["sh", "-c", "exec redis-server --requirepass \"$REDIS_PASSWORD\""]

dev/local/Makefile

@@ -64,8 +64,10 @@ CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 NOW:=$(shell date "+%Y-%m-%d")
 LOG_WEB:=delphi_web_epidata_$(NOW).log
 LOG_DB:=delphi_database_epidata_$(NOW).log
+LOG_REDIS:=delphi_redis_instance_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
+REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
 
 M1=
 ifeq ($(shell uname -smp), Darwin arm64 arm)
@@ -98,6 +100,10 @@ web:
 		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
+		--env "REDIS_HOST=delphi_redis" \
+		--env "REDIS_PASSWORD=1234" \
+		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
+		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 
@@ -136,8 +142,25 @@ py:
 		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
+.PHONY=redis
+redis:
+	@# Stop container if running
+	@if [ $(REDIS_CONTAINER_ID) ]; then\
+		docker stop $(REDIS_CONTAINER_ID);\
+	fi
+
+	@docker build -t delphi_redis \
+		$(M1) \
+		-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
+
+	@docker run --rm -d -p 127.0.0.1:6379:6379 \
+		$(M1) \
+		--network delphi-net \
+		--env "REDIS_PASSWORD=1234" \
+		--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &
+
 .PHONY=all
-all: db web py
+all: db web py redis
 
 .PHONY=test
 test:
@@ -149,6 +172,17 @@ test:
 		--env "FLASK_SECRET=abc" \
 		delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
 
+.PHONY=r-test
+r-test:
+	@docker run -i --rm --network delphi-net \
+		$(M1) \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
+		--env "FLASK_SECRET=abc" \
+		delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log
+
+
 .PHONY=bash
 bash:
 	@docker run -it --rm --network delphi-net \

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.12
+version = 4.1.0
 
 [options]
 packages =
@@ -28,6 +28,8 @@ packages =
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
     delphi.epidata.server
+    delphi.epidata.server.admin
+    delphi.epidata.server.admin.templates
     delphi.epidata.server.covidcast_issues_migration
     delphi.epidata.server.endpoints
     delphi.epidata.server.endpoints.covidcast_utils

docs/api/api_keys.md

@@ -16,11 +16,11 @@ change as we learn more about their impact:
 
 For example, a query for three signals on one date across all counties can be
 submitted anonymously, but a query for three signals on a period of four weeks
-across all counties requires an API key. 
+across all counties requires an API key.
 
 An API key is a pseudonymous access token that grants privileged access to the
-Epidata API. You can request an API key by 
-[registering with us](https://forms.gle/hkBr5SfQgxguAfEt7).
+Epidata API. You can request an API key by
+[registering with us](https://api.delphi.cmu.edu/epidata/admin/registration_form).
 Privileges of registration may include:
 
 1. no rate limit
@@ -36,7 +36,8 @@ store the information you provide us at registration time, see our
 
 ## Usage
 
-If you choose to [register for an API key](https://forms.gle/hkBr5SfQgxguAfEt7), 
+If you choose to
+[register for an API key](https://api.delphi.cmu.edu/epidata/admin/registration_form),
 there are several ways to use your key to authenticate your requests:
 
 ### Via request parameter

docs/api/privacy_statement.md

@@ -29,7 +29,7 @@ purposes:
 * to identify excessive or abnormal usage patterns which may harm our system
 
 The logs are only available to members of our operations team, and are expunged
-at or before they reach five years in age.  
+at or before they reach five years in age.
 
 If you provide us with your email address, we will only use it to contact you in
 the following scenarios:
@@ -47,9 +47,9 @@ security practices to help us keep your information secure. We only retrieve
 this mapping to resolve cases of excessive or abnormal usage. We automatically
 disassociate an email address from its API key when the API key has not been
 used for six months, or upon user request. You can request that your
-email address be removed from our records by filling out a 
-[deletion request](https://forms.gle/GucFmZHTMgEFjH197).
+email address be removed from our records by filling out a
+[deletion request](https://api.delphi.cmu.edu/epidata/admin/removal_request).
 
-For more information, see 
+For more information, see
 [Carnegie Mellon’s privacy notice](https://www.cmu.edu/legal/privacy-notice.html).
 Further questions can be directed to [email protected].