diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index e16c9c28..00000000 --- a/.bandit.yml +++ /dev/null @@ -1,396 +0,0 @@ - -### Bandit config file generated from: - -### This config may optionally select a subset of tests to run or skip by -### filling out the 'tests' and 'skips' lists given below. If no tests are -### specified for inclusion then it is assumed all tests are desired. The skips -### set will remove specific tests from the include set. This can be controlled -### using the -t/-s CLI options. Note that the same test ID should not appear -### in both 'tests' and 'skips', this would be nonsensical and is detected by -### Bandit at runtime. - -# Available tests: -# B101 : assert_used -# B102 : exec_used -# B103 : set_bad_file_permissions -# B104 : hardcoded_bind_all_interfaces -# B105 : hardcoded_password_string -# B106 : hardcoded_password_funcarg -# B107 : hardcoded_password_default -# B108 : hardcoded_tmp_directory -# B110 : try_except_pass -# B112 : try_except_continue -# B201 : flask_debug_true -# B301 : pickle -# B302 : marshal -# B303 : md5 -# B304 : ciphers -# B305 : cipher_modes -# B306 : mktemp_q -# B307 : eval -# B308 : mark_safe -# B309 : httpsconnection -# B310 : urllib_urlopen -# B311 : random -# B312 : telnetlib -# B313 : xml_bad_cElementTree -# B314 : xml_bad_ElementTree -# B315 : xml_bad_expatreader -# B316 : xml_bad_expatbuilder -# B317 : xml_bad_sax -# B318 : xml_bad_minidom -# B319 : xml_bad_pulldom -# B320 : xml_bad_etree -# B321 : ftplib -# B322 : input -# B323 : unverified_context -# B324 : hashlib_new_insecure_functions -# B325 : tempnam -# B401 : import_telnetlib -# B402 : import_ftplib -# B403 : import_pickle -# B404 : import_subprocess -# B405 : import_xml_etree -# B406 : import_xml_sax -# B407 : import_xml_expat -# B408 : import_xml_minidom -# B409 : import_xml_pulldom -# B410 : import_lxml -# B411 : import_xmlrpclib -# B412 : import_httpoxy -# B413 : import_pycrypto -# B501 : request_with_no_cert_validation -# B502 : ssl_with_bad_version -# B503 : ssl_with_bad_defaults -# B504 : ssl_with_no_version -# B505 : weak_cryptographic_key -# B506 : yaml_load -# B507 : ssh_no_host_key_verification -# B601 : paramiko_calls -# B602 : subprocess_popen_with_shell_equals_true -# B603 : subprocess_without_shell_equals_true -# B604 : any_other_function_with_shell_equals_true -# B605 : start_process_with_a_shell -# B606 : start_process_with_no_shell -# B607 : start_process_with_partial_path -# B608 : hardcoded_sql_expressions -# B609 : linux_commands_wildcard_injection -# B610 : django_extra_used -# B611 : django_rawsql_used -# B701 : jinja2_autoescape_false -# B702 : use_of_mako_templates -# B703 : django_mark_safe - -# (optional) list included test IDs here, eg '[B101, B406]': -tests: - -# (optional) list skipped test IDs here, eg '[B101, B406]': -skips: ['B104'] - -### (optional) plugin settings - some test plugins require configuration data -### that may be given here, per-plugin. All bandit test plugins have a built in -### set of sensible defaults and these will be used if no configuration is -### provided. It is not necessary to provide settings for every (or any) plugin -### if the defaults are acceptable. - -any_other_function_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -hardcoded_tmp_directory: - tmp_dirs: - - /tmp - - /var/tmp - - /dev/shm -linux_commands_wildcard_injection: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -ssl_with_bad_defaults: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD -ssl_with_bad_version: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD -start_process_with_a_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_no_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_partial_path: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_popen_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_without_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -try_except_continue: - check_typed_exception: false -try_except_pass: - check_typed_exception: false -weak_cryptographic_key: - weak_key_size_dsa_high: 1024 - weak_key_size_dsa_medium: 2048 - weak_key_size_ec_high: 160 - weak_key_size_ec_medium: 224 - weak_key_size_rsa_high: 1024 - weak_key_size_rsa_medium: 2048 - diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..b6978346 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,41 @@ +name: Documentation Checks + +on: [push] + +jobs: + spell_check: + strategy: + max-parallel: 4 + matrix: + os: [ubuntu-latest] + + runs-on: ${{ matrix.os }} + + steps: + - name: Spell check install + run: curl -L https://git.io/misspell | bash + - name: Spell check docs + run: bin/misspell -error docs/* + + code_docs: + strategy: + max-parallel: 4 + matrix: + os: [ubuntu-latest] + python-version: [3.7] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Run docs tests + run: tox -e docs + \ No newline at end of file diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml new file mode 100644 index 00000000..a3b70517 --- /dev/null +++ b/.github/workflows/int.yml @@ -0,0 +1,58 @@ +name: Integration Tests + +on: [pull_request] + +jobs: + build: + strategy: + max-parallel: 4 + matrix: + os: [ubuntu-latest] + python-version: [3.7] + + runs-on: ${{ matrix.os }} + + name: Integration Tests + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install requirements + run: | + wget https://github.com/openshift/source-to-image/releases/download/v1.2.0/source-to-image-v1.2.0-2a579ecd-linux-amd64.tar.gz + tar -xvf source-to-image-v1.2.0-2a579ecd-linux-amd64.tar.gz + sudo cp s2i /usr/local/bin + pip install aiohttp + pip install requests + - name: Build image + run: | + s2i build . centos/python-36-centos7 cscfi/beacon-python + + - name: Start Services + run: | + pushd deploy/test + docker-compose up -d + sleep 10 + docker exec test_beacon_1 beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz data/example_metadata.json + docker exec test_beacon_1 beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_registered.json + docker exec test_beacon_1 beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_controlled.json + docker exec test_beacon_1 beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_controlled1.json + + - name: Run Integration test + run: | + pushd deploy/test + python run_tests.py + + - name: Collect logs from docker + if: ${{ failure() }} + run: cd deploy && docker-compose logs --no-color -t > ../tests/dockerlogs || true + + - name: Persist log files + if: ${{ failure() }} + uses: actions/upload-artifact@v1 + with: + name: test_debugging_help + path: tests diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..e73dd097 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,57 @@ +name: Publish Docker image + +on: + release: + types: [published] + push: + branches: [master] + +jobs: + push_to_registry: + name: Push Beacon Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Login to DockerHub Registry + run: echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin + - name: Get the version + id: vars + run: echo ::set-output name=tag::$(echo ${GITHUB_REF:10}) + - name: Build the tagged Docker image + if: ${{ steps.vars.outputs.tag != '/master' }} + run: docker build . --file Dockerfile --tag cscfi/beacon-python:${{steps.vars.outputs.tag}} + - name: Push the tagged Docker image + if: ${{ steps.vars.outputs.tag != '/master' }} + run: docker push cscfi/beacon-python:${{steps.vars.outputs.tag}} + - name: Build the latest Docker image + if: ${{ steps.vars.outputs.tag == '/master' }} + run: docker build . --file Dockerfile --tag cscfi/beacon-python:latest + - name: Push the latest Docker image + if: ${{ steps.vars.outputs.tag == '/master' }} + run: docker push cscfi/beacon-python:latest + push_data_to_registry: + name: Push Dataloader Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Login to DockerHub Registry + run: echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin + - name: Get the version + id: vars + run: echo ::set-output name=tag::$(echo ${GITHUB_REF:10}) + - name: Build the tagged Docker image + if: ${{ steps.vars.outputs.tag != '/master' }} + run: | + pushd deploy/test + docker build . --file Dockerfile --tag cscfi/beacon-dataloader:${{steps.vars.outputs.tag}} + - name: Push the tagged Docker image + if: ${{ steps.vars.outputs.tag != '/master' }} + run: docker push cscfi/beacon-dataloader:${{steps.vars.outputs.tag}} + - name: Build the latest Docker image + if: ${{ steps.vars.outputs.tag == '/master' }} + run: | + pushd deploy/test + docker build . --file Dockerfile --tag cscfi/beacon-dataloader:latest + - name: Push the latest Docker image + if: ${{ steps.vars.outputs.tag == '/master' }} + run: docker push cscfi/beacon-dataloader:latest diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 00000000..6b77b67d --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,32 @@ +name: Python style check + +on: [push] + +jobs: + style_check: + strategy: + max-parallel: 4 + matrix: + os: [ubuntu-latest] + python-version: [3.6, 3.7] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Test flake8 syntax with tox + run: tox -e flake8 + - name: Do bandit static check with tox + run: tox -e bandit + - name: Install libcurl-devel + run: sudo apt-get install libcurl4-openssl-dev + - name: Do typing check with tox + run: tox -e mypy \ No newline at end of file diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml new file mode 100644 index 00000000..c42cab9e --- /dev/null +++ b/.github/workflows/unit.yml @@ -0,0 +1,30 @@ +name: Python Unit Tests + +on: [push] + +jobs: + unit_test: + strategy: + max-parallel: 4 + matrix: + os: [ubuntu-latest] + python-version: [3.6, 3.7] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install libcurl-devel + run: sudo apt-get install libcurl4-openssl-dev + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Run unit tests + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + run: tox -e unit_tests \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5300a3e2..00000000 --- a/.travis.yml +++ /dev/null @@ -1,98 +0,0 @@ -sudo: required -dist: xenial -language: python - -install: true - -git: - depth: false - quiet: true - -services: docker - -stages: - - name: tests - if: type IN (push, pull_request) - - name: integtests - if: type IN (pull_request) - - name: image - if: branch = master AND type = push - - name: image tag - if: tag =~ /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ - - -jobs: - include: - - stage: tests - name: "Code Style Check" - python: 3.6 - before_script: - - pip install tox-travis - script: tox -e flake8 - - stage: tests - name: "Unit Tests Python 3.6" - python: 3.6 - before_script: - - sudo apt-get update - - sudo apt-get install libcurl4-openssl-dev - - pip install tox-travis - script: tox -e py36 - - stage: tests - name: "Unit Tests Python 3.7" - python: 3.7 - before_script: - - sudo apt-get update - - sudo apt-get install libcurl4-openssl-dev - - pip install tox-travis - script: tox -e py37 - - stage: tests - name: "Documentation Tests" - python: 3.6 - before_script: - - pip install tox-travis - script: tox -e docs - - stage: tests - name: "Python Code Security Tests" - python: 3.6 - before_script: - - pip install tox-travis - script: tox -e bandit - - stage: integtests - name: "Integration Tests" - python: 3.6 - before_script: - - wget https://github.com/openshift/source-to-image/releases/download/v1.2.0/source-to-image-v1.2.0-2a579ecd-linux-amd64.tar.gz - - tar -xvf source-to-image-v1.2.0-2a579ecd-linux-amd64.tar.gz - - sudo cp s2i /usr/local/bin - - s2i build . centos/python-36-centos7 cscfi/beacon-python - - cd deploy/test - - docker-compose up -d - - sleep 10 - - docker-compose exec beacon beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz data/example_metadata.json - - docker-compose exec beacon beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_registered.json - - docker-compose exec beacon beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_controlled.json - - docker-compose exec beacon beacon_init data/ALL.chrMT.phase3_callmom-v0_4.20130502.genotypes.vcf.gz /exdata/example_metadata_controlled1.json - - pip install aiohttp - - pip install requests - script: - - python run_tests.py - - stage: image - name: "Image Publish" - before_script: - - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - script: - - docker build -t cscfi/beacon-python . - - docker push cscfi/beacon-python:latest - - cd deploy/dataloader - - docker build -t cscfi/beacon-dataloader . - - docker push cscfi/beacon-dataloader:latest - - stage: image tag - name: "Image Tag Publish" - before_script: - - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - script: - - docker build -t cscfi/beacon-python:$TRAVIS_TAG . - - docker push cscfi/beacon-python:$TRAVIS_TAG - -notifications: - email: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 322a1089..3379eb3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ Once submitted, the Pull Request will go through a review process, meaning we wi #### Git Branches We use `dev` branch as the main development branch and `master` as the releases branch. -All Pull Requests related to features should be done agains `dev` branch, releases Pull Requests should be done agains `master` branch. +All Pull Requests related to features should be done against `dev` branch, releases Pull Requests should be done against `master` branch. Give your branch a short descriptive name (like the names between the `<>` below) and prefix the name with something representative for that branch: diff --git a/README.md b/README.md index fbf4e103..48377215 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ ## beacon-python - Python-based Beacon API Web Server -[![Build Status](https://travis-ci.org/CSCfi/beacon-python.svg?branch=master)](https://travis-ci.org/CSCfi/beacon-python) -[![Coverage Status](https://coveralls.io/repos/github/CSCfi/beacon-python/badge.svg?branch=master)](https://coveralls.io/github/CSCfi/beacon-python?branch=master) +![Integration Tests](https://github.com/CSCfi/beacon-python/workflows/Integration%20Tests/badge.svg) +![Python Unit Tests](https://github.com/CSCfi/beacon-python/workflows/Python%20Unit%20Tests/badge.svg) +[![Coverage Status](https://coveralls.io/repos/github/CSCfi/beacon-python/badge.svg?branch=HEAD)](https://coveralls.io/github/CSCfi/beacon-python?branch=HEAD) [![Documentation Status](https://readthedocs.org/projects/beacon-python/badge/?version=latest)](https://beacon-python.readthedocs.io/en/latest/?badge=latest) -[![Docker Image](https://images.microbadger.com/badges/image/cscfi/beacon-python.svg)](https://microbadger.com/images/cscfi/beacon-python) Documentation: https://beacon-python.readthedocs.io diff --git a/beacon_api/api/exceptions.py b/beacon_api/api/exceptions.py index c538c137..43fc7cb1 100644 --- a/beacon_api/api/exceptions.py +++ b/beacon_api/api/exceptions.py @@ -4,13 +4,17 @@ """ import json +from typing import Dict from aiohttp import web from .. import __apiVersion__ from ..utils.logging import LOG from ..conf import CONFIG_INFO -def process_exception_data(request, host, error_code, error): +def process_exception_data(request: Dict, + host: str, + error_code: int, + error: str) -> Dict: """Return request data as dictionary. Generates custom exception messages based on request parameters. @@ -42,10 +46,11 @@ class BeaconBadRequest(web.HTTPBadRequest): """Exception returns with 400 code and a custom error message. The method is called if one of the required parameters are missing or invalid. - Used in conjuction with JSON Schema validator. + Used in conjunction with JSON Schema validator. """ - def __init__(self, request, host, error): + def __init__(self, request: Dict, + host: str, error: str) -> None: """Return custom bad request exception.""" data = process_exception_data(request, host, 400, error) super().__init__(text=json.dumps(data), content_type="application/json") @@ -56,10 +61,11 @@ class BeaconUnauthorised(web.HTTPUnauthorized): """HTTP Exception returns with 401 code with a custom error message. The method is called if the user is not registered or if the token from the authentication has expired. - Used in conjuction with Token authentication aiohttp middleware. + Used in conjunction with Token authentication aiohttp middleware. """ - def __init__(self, request, host, error, error_message): + def __init__(self, request: Dict, + host: str, error: str, error_message: str) -> None: """Return custom unauthorized exception.""" data = process_exception_data(request, host, 401, error) headers_401 = {"WWW-Authenticate": f"Bearer realm=\"{CONFIG_INFO.url}\"\n\ @@ -76,10 +82,11 @@ class BeaconForbidden(web.HTTPForbidden): `'Resource not granted for authenticated user or resource protected for all users.'`. The method is called if the dataset is protected or if the user is authenticated - but not granted the resource. Used in conjuction with Token authentication aiohttp middleware. + but not granted the resource. Used in conjunction with Token authentication aiohttp middleware. """ - def __init__(self, request, host, error): + def __init__(self, request: Dict, + host: str, error: str) -> None: """Return custom forbidden exception.""" data = process_exception_data(request, host, 403, error) super().__init__(content_type="application/json", text=json.dumps(data)) @@ -92,7 +99,7 @@ class BeaconServerError(web.HTTPInternalServerError): The 500 error is not specified by the Beacon API, thus as simple error would do. """ - def __init__(self, error): + def __init__(self, error: str) -> None: """Return custom forbidden exception.""" data = {'errorCode': 500, 'errorMessage': error} diff --git a/beacon_api/api/info.py b/beacon_api/api/info.py index 634a72d1..90aa4da4 100644 --- a/beacon_api/api/info.py +++ b/beacon_api/api/info.py @@ -6,6 +6,7 @@ .. note:: See ``beacon_api`` root folder ``__init__.py`` for changing values used here. """ +from typing import Dict from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__, __handover_beacon__ from .. import __createtime__, __updatetime__, __org_id__, __org_name__, __org_description__ from .. import __org_address__, __org_logoUrl__, __org_welcomeUrl__, __org_info__, __org_contactUrl__ @@ -17,7 +18,7 @@ @cached(ttl=60, key="ga4gh_info", serializer=JsonSerializer()) -async def ga4gh_info(host): +async def ga4gh_info(host: str) -> Dict: """Construct the `Beacon` app information dict in GA4GH Discovery format. :return beacon_info: A dict that contain information about the ``Beacon`` endpoint. @@ -43,7 +44,7 @@ async def ga4gh_info(host): @cached(ttl=60, key="info_key", serializer=JsonSerializer()) -async def beacon_info(host, pool): +async def beacon_info(host: str, pool) -> Dict: """Construct the `Beacon` app information dict. :return beacon_info: A dict that contain information about the ``Beacon`` endpoint. diff --git a/beacon_api/api/query.py b/beacon_api/api/query.py index cb01883c..2a02957f 100644 --- a/beacon_api/api/query.py +++ b/beacon_api/api/query.py @@ -5,6 +5,7 @@ start or end position. """ +from typing import Dict, Tuple, List, Optional from ..utils.logging import LOG from .. import __apiVersion__, __handover_beacon__, __handover_drs__ from ..utils.data_query import filter_exists, find_datasets, fetch_datasets_access @@ -13,7 +14,11 @@ from .exceptions import BeaconUnauthorised, BeaconForbidden, BeaconBadRequest -def access_resolution(request, token, host, public_data, registered_data, controlled_data): +def access_resolution(request: Dict, token: Dict, + host: str, + public_data: List[str], + registered_data: List[str], + controlled_data: List[str]) -> Tuple[List[str], List[str]]: """Determine the access level for a user. Depends on user bona_fide_status, and by default it should be PUBLIC. @@ -23,12 +28,12 @@ def access_resolution(request, token, host, public_data, registered_data, contro # unless the request is for specific datasets if public_data: permissions.append("PUBLIC") - access = set(public_data) # empty if no datasets are given + accessible_datasets = set(public_data) # empty if no datasets are given # for now we are expecting that the permissions are a list of datasets if registered_data and token["bona_fide_status"] is True: permissions.append("REGISTERED") - access = access.union(set(registered_data)) + accessible_datasets = accessible_datasets.union(set(registered_data)) # if user requests public datasets do not throw an error # if both registered and controlled datasets are request this will be shown first elif registered_data and not public_data: @@ -43,7 +48,7 @@ def access_resolution(request, token, host, public_data, registered_data, contro # Default event, when user doesn't specify dataset ids # Contains only dataset ids from token that are present at beacon controlled_access = set(controlled_data).intersection(set(token['permissions'])) - access = access.union(controlled_access) + accessible_datasets = accessible_datasets.union(controlled_access) if controlled_access: permissions.append("CONTROLLED") # if user requests public datasets do not throw an error @@ -54,11 +59,12 @@ def access_resolution(request, token, host, public_data, registered_data, contro raise BeaconUnauthorised(request, host, "missing_token", 'Unauthorized access to dataset(s), missing token.') # token is present, but is missing perms (user authed but no access) raise BeaconForbidden(request, host, 'Access to dataset(s) is forbidden.') - LOG.info(f"Accesible datasets are: {list(access)}.") - return permissions, list(access) + LOG.info(f"Accesible datasets are: {list(accessible_datasets)}.") + return permissions, list(accessible_datasets) -async def query_request_handler(params): + +async def query_request_handler(params: Tuple) -> Dict: """Handle the parameters of the query endpoint in order to find the required datasets. params = db_pool, method, request, token, host @@ -91,18 +97,20 @@ async def query_request_handler(params): raise BeaconBadRequest(request, params[4], "endMin value Must be smaller than endMax value") if request.get("startMin") and request.get("startMin") > request.get("startMax"): raise BeaconBadRequest(request, params[4], "startMin value Must be smaller than startMax value") - requested_position = (request.get("start", None), request.get("end", None), - request.get("startMin", None), request.get("startMax", None), - request.get("endMin", None), request.get("endMax", None)) + requested_position: Tuple[Optional[int], ...] = (request.get("start", None), request.get("end", None), + request.get("startMin", None), request.get("startMax", None), + request.get("endMin", None), request.get("endMax", None)) # Get dataset ids that were requested, sort by access level # If request is empty (default case) the three dataset variables contain all datasets by access level # Datasets are further filtered using permissions from token public_datasets, registered_datasets, controlled_datasets = await fetch_datasets_access(params[0], request.get("datasetIds")) - access_type, accessible_datasets = access_resolution(request, params[3], params[4], public_datasets, - registered_datasets, controlled_datasets) + access_type, accessible_datasets = access_resolution(request, + params[3], params[4], + public_datasets, registered_datasets, controlled_datasets) if 'mateName' in request or alleleRequest.get('variantType') == 'BND': - datasets = await find_fusion(params[0], request.get("assemblyId"), requested_position, request.get("referenceName"), + datasets = await find_fusion(params[0], + request.get("assemblyId"), requested_position, request.get("referenceName"), request.get("referenceBases"), request.get('mateName'), accessible_datasets, access_type, request.get("includeDatasetResponses", "NONE")) else: @@ -122,4 +130,5 @@ async def query_request_handler(params): if __handover_drs__: beacon_response['beaconHandover'] = make_handover(__handover_beacon__, [x['datasetId'] for x in datasets]) + return beacon_response diff --git a/beacon_api/app.py b/beacon_api/app.py index 04feba1a..a3603bce 100644 --- a/beacon_api/app.py +++ b/beacon_api/app.py @@ -13,7 +13,8 @@ from .conf.config import init_db_pool from .schemas import load_schema from .utils.logging import LOG -from .utils.validate import validate, token_auth, parse_request_object +from .utils.validate_json import validate, parse_request_object +from .utils.validate_jwt import token_auth import uvloop import asyncio import json @@ -27,7 +28,7 @@ # ---------------------------------------------------------------------------------------------------------------------- @routes.get('/') # For Beacon API Specification @routes.get('/service-info') # For GA4GH Discovery Specification -async def beacon_get(request): +async def beacon_get(request: web.Request) -> web.Response: """ Use the HTTP protocol 'GET' to return a Json object of all the necessary info on the beacon and the API. @@ -53,7 +54,7 @@ async def beacon_get(request): # These could be put under a @route.view('/query') @routes.get('/query') @validate(load_schema("query")) -async def beacon_get_query(request): +async def beacon_get_query(request: web.Request) -> web.Response: """Find datasets using GET endpoint.""" method, processed_request = await parse_request_object(request) params = request.app['pool'], method, processed_request, request["token"], request.host @@ -63,7 +64,7 @@ async def beacon_get_query(request): @routes.post('/query') @validate(load_schema("query")) -async def beacon_post_query(request): +async def beacon_post_query(request: web.Request) -> web.Response: """Find datasets using POST endpoint.""" method, processed_request = await parse_request_object(request) params = request.app['pool'], method, processed_request, request["token"], request.host @@ -71,7 +72,7 @@ async def beacon_post_query(request): return web.json_response(response, content_type='application/json', dumps=json.dumps) -async def initialize(app): +async def initialize(app: web.Application) -> None: """Spin up DB a connection pool with the HTTP server.""" # TO DO check if table and Database exist # and maybe exit gracefully or at least wait for a bit @@ -80,7 +81,7 @@ async def initialize(app): set_cors(app) -async def destroy(app): +async def destroy(app: web.Application) -> None: """Upon server close, close the DB connection pool.""" # will defer this to asyncpg await app['pool'].close() # pragma: no cover @@ -103,7 +104,7 @@ def set_cors(server): cors.add(route) -async def init(): +async def init() -> web.Application: """Initialise server.""" beacon = web.Application(middlewares=[token_auth()]) beacon.router.add_routes(routes) @@ -121,13 +122,13 @@ def main(): # sslcontext.load_cert_chain(ssl_certfile, ssl_keyfile) # sslcontext = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) # sslcontext.check_hostname = False - web.run_app(init(), host=os.environ.get('HOST', '0.0.0.0'), - port=os.environ.get('PORT', '5050'), + web.run_app(init(), host=os.environ.get('HOST', '0.0.0.0'), # nosec + port=os.environ.get('PORT', '5050'), # nosec shutdown_timeout=0, ssl_context=None) if __name__ == '__main__': if sys.version_info < (3, 6): - LOG.error("beacon-python requires python3.6") + LOG.error("beacon-python requires python 3.6") sys.exit(1) main() diff --git a/beacon_api/conf/__init__.py b/beacon_api/conf/__init__.py index 188f85cc..9128be6e 100644 --- a/beacon_api/conf/__init__.py +++ b/beacon_api/conf/__init__.py @@ -1,22 +1,30 @@ """Beacon Python Application Configuration.""" import json -import os +from os import environ from configparser import ConfigParser from collections import namedtuple from distutils.util import strtobool +from pathlib import Path +from typing import Any, Dict, List, Union -def parse_drspaths(paths): + +def convert(dictionary: Dict) -> tuple: + """Convert dictionary to Named tuple.""" + return namedtuple('Config', dictionary.keys())(**dictionary) + + +def parse_drspaths(paths: str) -> List[List[str]]: """Parse handover configuration.""" return [p.strip().split(',', 2) for p in paths.split('\n') if p.split()] -def parse_config_file(path): +def parse_config_file(path) -> Any: """Parse configuration file.""" config = ConfigParser() config.read(path) - config_vars = { + config_vars: Dict[str, Union[str, int, List[List[str]]]] = { 'title': config.get('beacon_general_info', 'title'), 'version': config.get('beacon_general_info', 'version'), 'author': config.get('beacon_general_info', 'author'), @@ -45,17 +53,17 @@ def parse_config_file(path): 'org_logoUrl': config.get('organisation_info', 'org_logoUrl'), 'org_info': config.get('organisation_info', 'org_info') } - return namedtuple("Config", config_vars.keys())(*config_vars.values()) + return convert(config_vars) -CONFIG_INFO = parse_config_file(os.environ.get('CONFIG_FILE', os.path.join(os.path.dirname(__file__), 'config.ini'))) +CONFIG_INFO = parse_config_file(environ.get('CONFIG_FILE', str(Path(__file__).resolve().parent.joinpath('config.ini')))) -def parse_oauth2_config_file(path): +def parse_oauth2_config_file(path: str) -> Any: """Parse configuration file.""" config = ConfigParser() config.read(path) - config_vars = { + config_vars: Dict[str, Union[str, bool, None]] = { 'server': config.get('oauth2', 'server'), 'issuers': config.get('oauth2', 'issuers'), 'userinfo': config.get('oauth2', 'userinfo'), @@ -63,10 +71,10 @@ def parse_oauth2_config_file(path): 'verify_aud': bool(strtobool(config.get('oauth2', 'verify_aud'))), 'bona_fide_value': config.get('oauth2', 'bona_fide_value') } - return namedtuple("Config", config_vars.keys())(*config_vars.values()) + return convert(config_vars) -OAUTH2_CONFIG = parse_oauth2_config_file(os.environ.get('CONFIG_FILE', os.path.join(os.path.dirname(__file__), 'config.ini'))) +OAUTH2_CONFIG = parse_oauth2_config_file(environ.get('CONFIG_FILE', str(Path(__file__).resolve().parent.joinpath('config.ini')))) # Sample query file should be of format [{BeaconAlleleRequest}] https://github.com/ga4gh-beacon/specification/ -sampleq_file = os.environ.get('SAMPLEQUERY_FILE', os.path.join(os.path.dirname(__file__), 'sample_queries.json')) -SAMPLE_QUERIES = json.load(open(sampleq_file)) if os.path.isfile(sampleq_file) else [] +sampleq_file = Path(environ.get('SAMPLEQUERY_FILE', str(Path(__file__).resolve().parent.joinpath('sample_queries.json')))) +SAMPLE_QUERIES = json.load(open(sampleq_file)) if sampleq_file.is_file() else [] diff --git a/beacon_api/conf/config.ini b/beacon_api/conf/config.ini index 2d97e0d6..8333cfad 100644 --- a/beacon_api/conf/config.ini +++ b/beacon_api/conf/config.ini @@ -7,7 +7,7 @@ title=GA4GHBeacon at CSC # Version of the Beacon implementation -version=1.6.1 +version=1.7.0 # Author of this software author=CSC developers diff --git a/beacon_api/conf/config.py b/beacon_api/conf/config.py index 2fd762cd..2570523a 100644 --- a/beacon_api/conf/config.py +++ b/beacon_api/conf/config.py @@ -16,11 +16,12 @@ import os import asyncpg +from typing import Awaitable DB_SCHEMA = os.environ.get('DATABASE_SCHEMA', None) -async def init_db_pool(): +async def init_db_pool() -> Awaitable: """Create a connection pool. As we will have frequent requests to the database it is recommended to create a connection pool. diff --git a/beacon_api/extensions/handover.py b/beacon_api/extensions/handover.py index 245caa8d..88a50eee 100644 --- a/beacon_api/extensions/handover.py +++ b/beacon_api/extensions/handover.py @@ -1,9 +1,10 @@ """Prepare Handover.""" +from typing import Dict, List from .. import __handover_drs__, __handover_datasets__, __handover_base__ -def add_handover(response): +def add_handover(response: Dict) -> Dict: """Add handover to a dataset response.""" response["datasetHandover"] = make_handover(__handover_datasets__, [response['datasetId']], response['referenceName'], response['start'], @@ -12,7 +13,11 @@ def add_handover(response): return response -def make_handover(paths, datasetIds, chr='', start=0, end=0, ref='', alt='', variant=''): +def make_handover(paths: List[List[str]], + datasetIds: List[str], + chr: str = '', + start: int = 0, end: int = 0, + ref: str = '', alt: str = '', variant: str = '') -> List[Dict]: """Create one handover for each path (specified in config).""" alt = alt if alt else variant handovers = [] diff --git a/beacon_api/extensions/mate_name.py b/beacon_api/extensions/mate_name.py index cd4c2afd..4a8328d6 100644 --- a/beacon_api/extensions/mate_name.py +++ b/beacon_api/extensions/mate_name.py @@ -5,6 +5,7 @@ from .handover import add_handover from ..utils.data_query import handle_wildcard, transform_misses, transform_record from .. import __handover_drs__ +from typing import Tuple, List, Optional async def fetch_fusion_dataset(db_pool, assembly_id, position, chromosome, reference, mate, @@ -31,7 +32,7 @@ async def fetch_fusion_dataset(db_pool, assembly_id, position, chromosome, refer refbase = None if not reference else handle_wildcard(reference) try: if misses: - # For MISS and ALL. We have already found all datasets with maching variants, + # For MISS and ALL. We have already found all datasets with matching variants, # so now just get one post per accessible, remaining datasets. query = """SELECT DISTINCT ON (datasetId) datasetId as "datasetId", accessType as "accessType", @@ -111,7 +112,13 @@ async def fetch_fusion_dataset(db_pool, assembly_id, position, chromosome, refer raise BeaconServerError(f'Query dataset DB error: {e}') -async def find_fusion(db_pool, assembly_id, position, chromosome, reference, mate, dataset_ids, access_type, include_dataset): +async def find_fusion(db_pool, + assembly_id: str, + position: Tuple[Optional[int], ...], + chromosome: str, reference: str, + mate: str, + dataset_ids: List[str], access_type: List, + include_dataset: str) -> List: """Find datasets based on filter parameters. This also takes into consideration the token value as to establish permissions. diff --git a/beacon_api/permissions/__init__.py b/beacon_api/permissions/__init__.py index b1b09934..1d91cbc7 100644 --- a/beacon_api/permissions/__init__.py +++ b/beacon_api/permissions/__init__.py @@ -6,7 +6,7 @@ that contains ``CONTROLLED`` permissions to datasets. Each file added in this module will consist of a function for parsing a permissions claim -from a JWT token. Then this function MUST be added to ``utils.validate`` module in order +from a JWT token. Then this function MUST be added to ``utils.validate_jwt`` module in order to pass permissions when retrieving datasets from the database. To avoid collisions check that your claim for permissions does not conflict with existing ones: diff --git a/beacon_api/permissions/ga4gh.py b/beacon_api/permissions/ga4gh.py index 73ff5512..a2d97ff1 100644 --- a/beacon_api/permissions/ga4gh.py +++ b/beacon_api/permissions/ga4gh.py @@ -83,17 +83,23 @@ """ import base64 import json +from typing import Dict, List, Tuple import aiohttp from authlib.jose import jwt +from authlib.jose import JWTClaims +from typing import Optional from ..api.exceptions import BeaconServerError from ..utils.logging import LOG from ..conf import OAUTH2_CONFIG -async def check_ga4gh_token(decoded_data, token, bona_fide_status, dataset_permissions): +async def check_ga4gh_token(decoded_data: JWTClaims, + token: Dict, + bona_fide_status: bool, + dataset_permissions: set) -> Tuple[set, bool]: """Check the token for GA4GH claims.""" LOG.debug('Checking GA4GH claims from scope.') @@ -107,7 +113,7 @@ async def check_ga4gh_token(decoded_data, token, bona_fide_status, dataset_permi return dataset_permissions, bona_fide_status -async def decode_passport(encoded_passport): +async def decode_passport(encoded_passport: str) -> List[Dict]: """Return decoded header and payload from encoded passport JWT. Public-key-less decoding inspired by the PyJWT library https://github.com/jpadilla/pyjwt @@ -115,8 +121,8 @@ async def decode_passport(encoded_passport): LOG.debug('Decoding GA4GH passport.') # Convert the token string into bytes for processing, and split it into segments - encoded_passport = encoded_passport.encode('utf-8') # `header.payload.signature` - data, _ = encoded_passport.rsplit(b'.', 1) # data contains header and payload segments, the ignored segment is the signature segment + decoded_passport = encoded_passport.encode('utf-8') # `header.payload.signature` + data, _ = decoded_passport.rsplit(b'.', 1) # data contains header and payload segments, the ignored segment is the signature segment segments = data.split(b'.', 1) # [header, payload] # Intermediary container @@ -139,12 +145,12 @@ async def decode_passport(encoded_passport): return decoded_data -async def get_ga4gh_permissions(token): +async def get_ga4gh_permissions(token: Dict) -> tuple: """Retrieve GA4GH passports (JWTs) from ELIXIR AAI and process them into tangible permissions.""" LOG.info('Handling permissions.') # Return variables - dataset_permissions = [] + dataset_permissions = set() bona_fide_status = False # Intermediary containers @@ -176,7 +182,7 @@ async def get_ga4gh_permissions(token): return dataset_permissions, bona_fide_status -async def retrieve_user_data(token): +async def retrieve_user_data(token: Dict) -> Optional[str]: """Retrieve GA4GH user data.""" LOG.debug('Contacting ELIXIR AAI /userinfo.') headers = {"Authorization": f"Bearer {token}"} @@ -190,7 +196,7 @@ async def retrieve_user_data(token): raise BeaconServerError("Could not retrieve GA4GH user data from ELIXIR AAI.") -async def get_jwk(url): +async def get_jwk(url: str) -> Optional[Dict]: """Get JWK set keys to validate JWT.""" LOG.debug('Retrieving JWK.') try: @@ -202,10 +208,10 @@ async def get_jwk(url): # This is not a fatal error, it just means that we are unable to validate the permissions, # but the process should continue even if the validation of one token fails LOG.error(f'Could not retrieve JWK from {url}') - pass + return None -async def validate_passport(passport): +async def validate_passport(passport: Dict) -> JWTClaims: """Decode a passport and validate its contents.""" LOG.debug('Validating passport.') @@ -245,7 +251,7 @@ async def validate_passport(passport): LOG.error(f"Something went wrong when processing JWT tokens: {e}") -async def get_ga4gh_controlled(passports): +async def get_ga4gh_controlled(passports: List) -> set: """Retrieve dataset permissions from GA4GH passport visas.""" # We only want to get datasets once, thus the set which prevents duplicates LOG.info("Parsing GA4GH dataset permissions.") @@ -264,7 +270,7 @@ async def get_ga4gh_controlled(passports): return datasets -async def get_ga4gh_bona_fide(passports): +async def get_ga4gh_bona_fide(passports: List) -> bool: """Retrieve Bona Fide status from GA4GH JWT claim.""" LOG.info("Parsing GA4GH bona fide claims.") diff --git a/beacon_api/schemas/__init__.py b/beacon_api/schemas/__init__.py index f481ab28..9a6b7d10 100644 --- a/beacon_api/schemas/__init__.py +++ b/beacon_api/schemas/__init__.py @@ -9,16 +9,17 @@ * ``response.json`` - beacon API JSON response. """ -import os import json +from typing import Dict +from pathlib import Path -def load_schema(name): +def load_schema(name: str) -> Dict: """Load JSON schemas.""" - module_path = os.path.dirname(__file__) - path = os.path.join(module_path, '{0}.json'.format(name)) + module_path = Path(__file__).resolve().parent + path = module_path.joinpath(f'{name}.json') - with open(os.path.abspath(path), 'r') as fp: + with open(str(path), 'r') as fp: data = fp.read() return json.loads(data) diff --git a/beacon_api/schemas/info.json b/beacon_api/schemas/info.json index dfb20f2a..a8e5abfe 100644 --- a/beacon_api/schemas/info.json +++ b/beacon_api/schemas/info.json @@ -1,7 +1,459 @@ { - "definitions": {}, "type": "object", "additionalProperties": false, + "definitions": { + "string": { + "$id": "$/definitions/string", + "type": "string", + "pattern": "^(.*)$" + }, + "orgType": { + "$id": "$/definitions/orgType", + "type": "string", + "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] + }, + "orgPurpose": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "$ref": "#/definitions/string" + }, + "obligatory": { + "type": "boolean" + } + } + } + }, + "consentDataUse": { + "$id": "$/definitions/consentDataUse", + "type": "object", + "required": [ + "primaryCategory", + "version" + ], + "properties": { + "primaryCategory": { + "type": "object", + "required": [ + "code" + ], + "properties": { + "code": { + "$ref": "#/definitions/string" + }, + "description": { + "$ref": "#/definitions/string" + } + } + }, + "secondaryCategories": { + "type": "array", + "items": { + "type": "object", + "required": [ + "code" + ], + "properties": { + "code": { + "$ref": "#/definitions/string" + }, + "description": { + "$ref": "#/definitions/string" + } + } + } + }, + "requirements": { + "type": "array", + "items": { + "type": "object", + "required": [ + "code" + ], + "properties": { + "code": { + "$ref": "#/definitions/string" + }, + "description": { + "$ref": "#/definitions/string" + } + } + } + }, + "version": { + "type": "string" + } + } + }, + "adamDataUse": { + "$id":"$/definitions/adamDataUse", + "type": "object", + "required": [ + "header", + "profile", + "terms", + "metaConditions" + ], + "properties": { + "header": { + "type": "object", + "properties": { + "matrixName": { + "$ref": "#/definitions/string" + }, + "matrixVersion": { + "$ref": "#/definitions/string" + }, + "matrixReferences": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "matrixProfileCreateDate": { + "$ref": "#/definitions/string" + }, + "matrixProfileUpdates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "$ref": "#/definitions/string" + }, + "description": { + "$ref": "#/definitions/string" + } + } + } + }, + "resourceName": { + "$ref": "#/definitions/string" + }, + "resourceReferences": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "resourceDescription": { + "$ref": "#/definitions/string" + }, + "resourceDataLevel": { + "type": "string", + "enum": ["UNKNOWN", "DATABASE", "METADATA", + "SUMMARISED", "DATASET", "RECORDSET", "RECORD", "RECORDFIELD"] + }, + "resourceContactNames": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/string" + }, + "email": { + "$ref": "#/definitions/string" + } + } + } + }, + "resourceContactOrganisations": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + } + } + }, + "profile": { + "type": "object", + "properties": { + "country": { + "type": "string", + "enum": ["UNRESTRICTED", "LIMITED"] + }, + "allowedCountries": { + "$ref": "#/definitions/orgPurpose" + }, + "organisation": { + "type": "string", + "enum": ["UNRESTRICTED", "LIMITED"] + }, + "allowedOrganisations": { + "$ref": "#/definitions/orgPurpose" + }, + "nonProfitOrganisation": { + "$ref": "#/definitions/orgType" + }, + "allowedNonProfitOrganisations": { + "$ref": "#/definitions/orgPurpose" + }, + "profitOrganisation": { + "$ref": "#/definitions/orgType" + }, + "allowedProfitOrganisations": { + "$ref": "#/definitions/orgPurpose" + }, + "person": { + "type": "string", + "enum": ["UNRESTRICTED", "LIMITED"] + }, + "allowedPersons": { + "$ref": "#/definitions/orgPurpose" + }, + "academicProfessional": { + "$ref": "#/definitions/orgType" + }, + "allowedAcademicProfessionals": { + "$ref": "#/definitions/orgPurpose" + }, + "clinicalProfessional": { + "$ref": "#/definitions/orgType" + }, + "allowedClinicalProfessionals": { + "$ref": "#/definitions/orgPurpose" + }, + "profitProfessional": { + "$ref": "#/definitions/orgType" + }, + "allowedProfitProfessionals": { + "$ref": "#/definitions/orgPurpose" + }, + "nonProfessional": { + "$ref": "#/definitions/orgType" + }, + "allowedNonProfessionals": { + "$ref": "#/definitions/orgPurpose" + }, + "nonProfitPurpose": { + "$ref": "#/definitions/orgType" + }, + "allowedNonProfitPurposes": { + "$ref": "#/definitions/orgPurpose" + }, + "profitPurpose": { + "$ref": "#/definitions/orgType" + }, + "allowedProfitPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "$ref": "#/definitions/string" + }, + "obligatory": { + "type": "boolean" + } + } + } + }, + "researchPurpose": { + "$ref": "#/definitions/orgType" + }, + "allowedResearchPurposes": { + "$ref": "#/definitions/orgPurpose" + }, + "allowedResearchProfiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum" : [ "OTHER", "METHODS", "CONTROL", "POPULATION", "ANCESTRY", "BIOMEDICAL", "FUNDAMENTAL", "GENETIC", "DRUG", "DISEASE", "GENDER", "AGE" ] + }, + "description": { + "$ref": "#/definitions/string" + }, + "restriction": { + "$ref": "#/definitions/orgType" + } + } + } + }, + "clinicalPurpose": { + "$ref": "#/definitions/string" + }, + "allowedClinicalPurpose": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "$ref": "#/definitions/string" + }, + "obligatory": { + "type": "boolean" + } + } + } + }, + "allowedClinicalProfiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum" : [ "OTHER", "DECISION_SUPPORT", "DISEASE" ] + }, + "description": { + "$ref": "#/definitions/string" + }, + "restriction": { + "type": "string", + "enum" : [ "OTHER", "DECISION_SUPPORT", "DISEASE" ] + } + } + } + } + } + }, + "terms": { + "type": "object", + "properties": { + "noAuthorizationTerms": { + "type": "boolean" + }, + "whichAuthorizationTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noPublicationTerms": { + "type": "boolean" + }, + "whichPublicationTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noTimelineTerms": { + "type": "boolean" + }, + "whichTimelineTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noSecurityTerms": { + "type": "boolean" + }, + "whichSecurityTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noExpungingTerms": { + "type": "boolean" + }, + "whichExpungingTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noLinkingTerms": { + "type": "boolean" + }, + "whichLinkingTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noRecontactTerms": { + "type": "boolean" + }, + "allowedRecontactTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "compulsoryRecontactTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noIPClaimTerms": { + "type": "boolean" + }, + "whichIPClaimTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noReportingTerms": { + "type": "boolean" + }, + "whichReportingTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noCollaborationTerms": { + "type": "boolean" + }, + "whichCollaborationTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "noPaymentTerms": { + "type": "boolean" + }, + "whichPaymentTerms": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + } + } + }, + "metaConditions": { + "type": "object", + "properties": { + "sharingMode": { + "type": "string", + "enum" : [ "UNKNOWN", "DISCOVERY", "ACCESS", "DISCOVERY_AND_ACCESS" ] + }, + "multipleObligationsRule": { + "type": "string", + "enum" : [ "MEET_ALL_OBLIGATIONS", "MEET_AT_LEAST_ONE_OBLIGATION" ] + }, + "noOtherConditions": { + "type": "boolean" + }, + "whichOtherConditions": { + "type": "array", + "items": { + "$ref": "#/definitions/string" + } + }, + "sensitivePopulations": { + "type": "boolean" + }, + "uniformConsent": { + "type": "boolean" + } + } + } + } + } + }, "required": [ "id", "name", @@ -11,16 +463,13 @@ ], "properties": { "id": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "name": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "apiVersion": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "organization": { "type": "object", @@ -30,32 +479,25 @@ ], "properties": { "id": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "name": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "description": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "address": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "welcomeUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "contactUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "logoUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "info": { "type": "object" @@ -63,28 +505,22 @@ } }, "description": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "version": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "welcomeUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "alternativeUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "createDateTime": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "updateDateTime": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "datasets": { "type": "array", @@ -99,32 +535,26 @@ ], "properties": { "id": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "name": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "description": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "assemblyId": { "type": "string", "pattern": "^((GRCh|hg)[0-9]+([.]?p[0-9]+)?)$" }, "createDateTime": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "updateDateTime": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "version": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "variantCount": { "type": "integer", @@ -153,607 +583,10 @@ ], "properties": { "consentCodeDataUse": { - "type": "object", - "required": [ - "primaryCategory", - "version" - ], - "properties": { - "primaryCategory": { - "type": "object", - "required": [ - "code" - ], - "properties": { - "code": { - "type": "string", - "pattern": "^(.*)$" - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - } - } - }, - "secondaryCategories": { - "type": "array", - "items": { - "type": "object", - "required": [ - "code" - ], - "properties": { - "code": { - "type": "string", - "pattern": "^(.*)$" - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "requirements": { - "type": "array", - "items": { - "type": "object", - "required": [ - "code" - ], - "properties": { - "code": { - "type": "string", - "pattern": "^(.*)$" - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "version": { - "type": "string" - } - } + "$ref": "#/definitions/consentDataUse" }, "adamDataUse": { - "type": "object", - "required": [ - "header", - "profile", - "terms", - "metaConditions" - ], - "properties": { - "header": { - "type": "object", - "properties": { - "matrixName": { - "type": "string", - "pattern": "^(.*)$" - }, - "matrixVersion": { - "type": "string", - "pattern": "^(.*)$" - }, - "matrixReferences": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "matrixProfileCreateDate": { - "type": "string", - "pattern": "^(.*)$" - }, - "matrixProfileUpdates": { - "type": "array", - "items": { - "type": "object", - "properties": { - "date": { - "type": "string", - "pattern": "^(.*)$" - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "resourceName": { - "type": "string", - "pattern": "^(.*)$" - }, - "resourceReferences": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "resourceDescription": { - "type": "string", - "pattern": "^(.*)$" - }, - "resourceDataLevel": { - "type": "string", - "enum": ["UNKNOWN", "DATABASE", "METADATA", - "SUMMARISED", "DATASET", "RECORDSET", "RECORD", "RECORDFIELD"] - }, - "resourceContactNames": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "pattern": "^(.*)$" - }, - "email": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "resourceContactOrganisations": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "profile": { - "type": "object", - "properties": { - "country": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED"] - }, - "allowedCountries": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "organisation": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED"] - }, - "allowedOrganisations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "nonProfitOrganisation": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedNonProfitOrganisations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "profitOrganisation": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedProfitOrganisations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "person": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED"] - }, - "allowedPersons": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "academicProfessional": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedAcademicProfessionals": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "clinicalProfessional": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedClinicalProfessionals": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "profitProfessional": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedProfitProfessionals": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "nonProfessional": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedNonProfessionals": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "nonProfitPurpose": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedNonProfitPurposes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "profitPurpose": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedProfitPurposes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "researchPurpose": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - }, - "allowedResearchPurposes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "allowedResearchProfiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum" : [ "OTHER", "METHODS", "CONTROL", "POPULATION", "ANCESTRY", "BIOMEDICAL", "FUNDAMENTAL", "GENETIC", "DRUG", "DISEASE", "GENDER", "AGE" ] - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "restriction": { - "type": "string", - "enum": ["UNRESTRICTED", "LIMITED", "UNRESTRICTED_OBLIGATORY", "LIMITED_OBLIGATORY", "FORBIDDEN"] - } - } - } - }, - "clinicalPurpose": { - "type": "string", - "pattern": "^(.*)$" - }, - "allowedClinicalPurpose": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "obligatory": { - "type": "boolean" - } - } - } - }, - "allowedClinicalProfiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum" : [ "OTHER", "DECISION_SUPPORT", "DISEASE" ] - }, - "description": { - "type": "string", - "pattern": "^(.*)$" - }, - "restriction": { - "type": "string", - "enum" : [ "OTHER", "DECISION_SUPPORT", "DISEASE" ] - } - } - } - } - } - }, - "terms": { - "type": "object", - "properties": { - "noAuthorizationTerms": { - "type": "boolean" - }, - "whichAuthorizationTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noPublicationTerms": { - "type": "boolean" - }, - "whichPublicationTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noTimelineTerms": { - "type": "boolean" - }, - "whichTimelineTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noSecurityTerms": { - "type": "boolean" - }, - "whichSecurityTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noExpungingTerms": { - "type": "boolean" - }, - "whichExpungingTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noLinkingTerms": { - "type": "boolean" - }, - "whichLinkingTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noRecontactTerms": { - "type": "boolean" - }, - "allowedRecontactTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "compulsoryRecontactTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noIPClaimTerms": { - "type": "boolean" - }, - "whichIPClaimTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noReportingTerms": { - "type": "boolean" - }, - "whichReportingTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noCollaborationTerms": { - "type": "boolean" - }, - "whichCollaborationTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "noPaymentTerms": { - "type": "boolean" - }, - "whichPaymentTerms": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - } - } - }, - "metaConditions": { - "type": "object", - "properties": { - "sharingMode": { - "type": "string", - "enum" : [ "UNKNOWN", "DISCOVERY", "ACCESS", "DISCOVERY_AND_ACCESS" ] - }, - "multipleObligationsRule": { - "type": "string", - "enum" : [ "MEET_ALL_OBLIGATIONS", "MEET_AT_LEAST_ONE_OBLIGATION" ] - }, - "noOtherConditions": { - "type": "boolean" - }, - "whichOtherConditions": { - "type": "array", - "items": { - "type": "string", - "pattern": "^(.*)$" - } - }, - "sensitivePopulations": { - "type": "boolean" - }, - "uniformConsent": { - "type": "boolean" - } - } - } - } + "$ref": "#/definitions/adamDataUse" } } } diff --git a/beacon_api/schemas/query.json b/beacon_api/schemas/query.json index ebf1270c..dd475b1c 100644 --- a/beacon_api/schemas/query.json +++ b/beacon_api/schemas/query.json @@ -1,7 +1,25 @@ { - "definitions": {}, "type": "object", "additionalProperties": false, + "definitions": { + "chromosome": { + "$id": "$/definitions/chromosome", + "type": "string", + "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", + "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" + ] + }, + "integer": { + "$id": "$/definitions/integer", + "type": "integer", + "minimum": 0 + }, + "variantType": { + "$id": "$/definitions/variantType", + "type": "string", + "enum": ["DEL", "INS", "DUP", "INV", "CNV", "SNP", "MNP", "DUP:TANDEM", "DEL:ME", "INS:ME", "BND"] + } + }, "required": [ "referenceName", "referenceBases", @@ -9,40 +27,28 @@ ], "properties": { "referenceName": { - "type": "string", - "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" - ] + "$ref": "#/definitions/chromosome" }, "mateName": { - "type": "string", - "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" - ] + "$ref": "#/definitions/chromosome" }, "start": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "end": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "startMin": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "startMax": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "endMin": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "endMax": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "referenceBases": { "type": "string", @@ -53,8 +59,7 @@ "pattern": "^([ACGTN]+)$" }, "variantType": { - "type": "string", - "enum": ["DEL", "INS", "DUP", "INV", "CNV", "SNP", "MNP", "DUP:TANDEM", "DEL:ME", "INS:ME", "BND"] + "$ref": "#/definitions/variantType" }, "assemblyId": { "type": "string", diff --git a/beacon_api/schemas/response.json b/beacon_api/schemas/response.json index c02b05fb..d0d8d035 100644 --- a/beacon_api/schemas/response.json +++ b/beacon_api/schemas/response.json @@ -1,120 +1,26 @@ { - "definitions": {}, "type": "object", "additionalProperties": false, - "required": [ - "beaconId" - ], - "properties": { - "beaconId": { - "type": "string", - "pattern": "^(.*)$" - }, - "apiVersion": { + "definitions": { + "chromosome": { + "$id": "$/definitions/chromosome", "type": "string", - "pattern": "^(.*)$" - }, - "exists": { - "oneOf": [{ - "type": "boolean" - }, - { - "type": "null" - } + "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", + "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" ] }, - "alleleRequest": { - "type": "object", - "required": [ - "referenceName", - "referenceBases", - "assemblyId" - ], - "properties": { - "referenceName": { - "oneOf": [{ - "type": "null" - }, - { - "type": "string", - "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" - ] - } - ] - }, - "mateName": { - "type": "string", - "enum": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y", "MT" - ] - }, - "start": { - "type": "integer", - "minimum": 0 - }, - "end": { - "type": "integer", - "minimum": 0 - }, - "startMin": { - "type": "integer", - "minimum": 0 - }, - "startMax": { - "type": "integer", - "minimum": 0 - }, - "endMin": { - "type": "integer", - "minimum": 0 - }, - "endMax": { - "type": "integer", - "minimum": 0 - }, - "referenceBases": { - "oneOf": [{ - "type": "null" - }, - { - "type": "string", - "pattern": "^([ACGTN]+)$" - } - ] - }, - "alternateBases": { - "type": "string", - "pattern": "^([ACGTN]+)$" - }, - "variantType": { - "type": "string", - "enum": ["DEL", "INS", "DUP", "INV", "CNV", "SNP", "MNP", "DUP:TANDEM", "DEL:ME", "INS:ME", "BND"] - }, - "assemblyId": { - "oneOf": [{ - "type": "null" - }, - { - "type": "string", - "pattern": "^((GRCh|hg)[0-9]+([.]?p[0-9]+)?)$" - } - ] - }, - "datasetIds": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[^<>'\"/;`%{}+=]*$" - } - }, - "includeDatasetResponses": { - "type": "string", - "enum": ["ALL", "HIT", "MISS", "NONE"] - } - } + "integer": { + "$id": "$/definitions/integer", + "type": "integer", + "minimum": 0 }, - "beaconHandover": { + "variantType": { + "$id": "$/definitions/variantType", + "type": "string", + "enum": ["DEL", "INS", "DUP", "INV", "CNV", "SNP", "MNP", "DUP:TANDEM", "DEL:ME", "INS:ME", "BND"] + }, + "handover": { + "$id": "$/definitions/handover", "type": "array", "required": [ "handoverType", @@ -136,14 +42,14 @@ } }, "description": { - "type": "string" + "type": "string" }, "url": { - "type": "string" + "type": "string" } } }, - "datasetAlleleResponses": { + "datasetAlleleResponse": { "type": "array", "items": { "type": "object", @@ -156,7 +62,8 @@ "pattern": "^(.*)$" }, "exists": { - "oneOf": [{ + "oneOf": [ + { "type": "boolean" }, { @@ -165,33 +72,7 @@ ] }, "datasetHandover": { - "type": "array", - "required": [ - "handoverType", - "url" - ], - "properties": { - "handoverType": { - "type": "object", - "required": [ - "id" - ], - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - } - } - }, - "description": { - "type": "string" - }, - "url": { - "type": "string" - } - } + "$ref": "#/definitions/handover" }, "referenceBases": { "type": "string", @@ -208,8 +89,7 @@ "type": "integer" }, "variantType": { - "type": "string", - "enum": ["DEL", "INS", "DUP", "INV", "CNV", "SNP", "MNP", "DUP:TANDEM", "DEL:ME", "INS:ME", "BND"] + "$ref": "#/definitions/variantType" }, "error": { "type": "object", @@ -232,16 +112,13 @@ "maximum": 1 }, "variantCount": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "callCount": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "sampleCount": { - "type": "integer", - "minimum": 0 + "$ref": "#/definitions/integer" }, "note": { "type": "string", @@ -256,16 +133,141 @@ } }, "if": { - "properties": { "exists": { "type": "null" } } + "properties": { + "exists": { + "type": "null" + } + } }, "then": { - "required": ["error"] + "required": [ + "error" + ] }, "else": { - "not": {"required": ["error"]} + "not": { + "required": [ + "error" + ] + } + } + } + } + }, + "required": [ + "beaconId" + ], + "properties": { + "beaconId": { + "type": "string", + "pattern": "^(.*)$" + }, + "apiVersion": { + "type": "string", + "pattern": "^(.*)$" + }, + "exists": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "alleleRequest": { + "type": "object", + "required": [ + "referenceName", + "referenceBases", + "assemblyId" + ], + "properties": { + "referenceName": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/definitions/chromosome" + } + ] + }, + "mateName": { + "$ref": "#/definitions/chromosome" + }, + "start": { + "$ref": "#/definitions/integer" + }, + "end": { + "$ref": "#/definitions/integer" + }, + "startMin": { + "$ref": "#/definitions/integer" + }, + "startMax": { + "$ref": "#/definitions/integer" + }, + "endMin": { + "$ref": "#/definitions/integer" + }, + "endMax": { + "$ref": "#/definitions/integer" + }, + "referenceBases": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^([ACGTN]+)$" + } + ] + }, + "alternateBases": { + "type": "string", + "pattern": "^([ACGTN]+)$" + }, + "variantType": { + "$ref": "#/definitions/variantType" + }, + "assemblyId": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^((GRCh|hg)[0-9]+([.]?p[0-9]+)?)$" + } + ] + }, + "datasetIds": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^<>'\"/;`%{}+=]*$" + } + }, + "includeDatasetResponses": { + "type": "string", + "enum": [ + "ALL", + "HIT", + "MISS", + "NONE" + ] } } }, + "beaconHandover": { + "$ref": "#/definitions/handover" + }, + "datasetAlleleResponses": { + "$ref": "#/definitions/datasetAlleleResponse" + }, "error": { "type": "object", "required": [ @@ -283,12 +285,22 @@ } }, "if": { - "properties": { "exists": { "type": "null" } } + "properties": { + "exists": { + "type": "null" + } + } }, "then": { - "required": ["error"] + "required": [ + "error" + ] }, "else": { - "not": {"required": ["error"]} + "not": { + "required": [ + "error" + ] + } } -} +} \ No newline at end of file diff --git a/beacon_api/schemas/service-info.json b/beacon_api/schemas/service-info.json index fec430ce..4b221229 100644 --- a/beacon_api/schemas/service-info.json +++ b/beacon_api/schemas/service-info.json @@ -1,7 +1,13 @@ { - "definitions": {}, "type": "object", "additionalProperties": false, + "definitions": { + "string": { + "$id": "$/definitions/string", + "type": "string", + "pattern": "^(.*)$" + } + }, "required": [ "id", "name", @@ -12,12 +18,10 @@ ], "properties": { "id": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "name": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "type": { "type": "object", @@ -28,22 +32,18 @@ ], "properties": { "group": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "artifact": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "version": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" } } }, "description": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "organization": { "type": "object", @@ -53,38 +53,30 @@ ], "properties": { "name": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "url": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" } } }, "contactUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "documentationUrl": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "createdAt": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "updatedAt": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "environment": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" }, "version": { - "type": "string", - "pattern": "^(.*)$" + "$ref": "#/definitions/string" } } } diff --git a/beacon_api/utils/data_query.py b/beacon_api/utils/data_query.py index 8583f7e0..0a2c0819 100644 --- a/beacon_api/utils/data_query.py +++ b/beacon_api/utils/data_query.py @@ -1,14 +1,17 @@ -"""Query DB and prepare data for reponse.""" +"""Query DB and prepare data for response.""" from datetime import datetime from functools import partial +from typing import Dict, List, Optional + +from typing import Tuple from .logging import LOG from ..api.exceptions import BeaconServerError from ..extensions.handover import add_handover from .. import __handover_drs__ -def transform_record(record): +def transform_record(record) -> Dict: """Format the record we got from the database to adhere to the response schema.""" response = dict(record) response["referenceBases"] = response.pop("referenceBases") # NOT part of beacon specification @@ -27,7 +30,7 @@ def transform_record(record): return response -def transform_misses(record): +def transform_misses(record) -> Dict: """Format the missed datasets record we got from the database to adhere to the response schema.""" response = dict(record) response["referenceBases"] = '' # NOT part of beacon specification @@ -48,7 +51,7 @@ def transform_misses(record): return response -def transform_metadata(record): +def transform_metadata(record) -> Dict: """Format the metadata record we got from the database to adhere to the response schema.""" response = dict(record) response["info"] = {"accessType": response.pop("accessType")} @@ -60,7 +63,7 @@ def transform_metadata(record): return response -async def fetch_datasets_access(db_pool, datasets): +async def fetch_datasets_access(db_pool, datasets: Optional[List]): """Retrieve CONTROLLED datasets.""" public = [] registered = [] @@ -120,7 +123,7 @@ async def fetch_dataset_metadata(db_pool, datasets=None, access_type=None): raise BeaconServerError(f'Query metadata DB error: {e}') -def handle_wildcard(sequence): +def handle_wildcard(sequence) -> List: """Construct PostgreSQL friendly wildcard string.""" if 'N' in sequence: # Wildcard(s) found, use wildcard notation @@ -156,7 +159,7 @@ async def fetch_filtered_dataset(db_pool, assembly_id, position, chromosome, ref refbase = None if not reference else handle_wildcard(reference) try: if misses: - # For MISS and ALL. We have already found all datasets with maching variants, + # For MISS and ALL. We have already found all datasets with matching variants, # so now just get one post per accessible, remaining datasets. query = """SELECT DISTINCT ON (datasetId) datasetId as "datasetId", accessType as "accessType", @@ -214,22 +217,30 @@ async def fetch_filtered_dataset(db_pool, assembly_id, position, chromosome, ref raise BeaconServerError(f'Query dataset DB error: {e}') -def filter_exists(include_dataset, datasets): +def filter_exists(include_dataset: str, datasets: List) -> List[str]: """Return those datasets responses that the `includeDatasetResponses` parameter decides. Look at the exist parameter in each returned dataset to established HIT or MISS. """ + data = [] if include_dataset == 'ALL': - return datasets + data = datasets elif include_dataset == 'NONE': - return [] + data = [] elif include_dataset == 'HIT': - return [d for d in datasets if d['exists'] is True] + data = [d for d in datasets if d['exists'] is True] elif include_dataset == 'MISS': - return [d for d in datasets if d['exists'] is False] + data = [d for d in datasets if d['exists'] is False] + + return data -async def find_datasets(db_pool, assembly_id, position, chromosome, reference, alternate, dataset_ids, access_type, include_dataset): +async def find_datasets(db_pool, + assembly_id: str, + position: Tuple[Optional[int], ...], + chromosome: str, reference: str, alternate: Tuple, + dataset_ids: List[str], access_type: List, + include_dataset: str) -> List: """Find datasets based on filter parameters. This also takes into consideration the token value as to establish permissions. diff --git a/beacon_api/utils/db_load.py b/beacon_api/utils/db_load.py index 679cdfa6..821c194a 100644 --- a/beacon_api/utils/db_load.py +++ b/beacon_api/utils/db_load.py @@ -54,7 +54,7 @@ class BeaconDB: """Database connection and operations.""" - def __init__(self): + def __init__(self) -> None: """Start database routines.""" LOG.info('Start database routines') self._conn = None @@ -230,7 +230,7 @@ async def load_metadata(self, vcf, metafile, datafile): LOG.info(metadata) LOG.info('Metadata has been parsed') try: - LOG.info(f'Attempting to insert metadata to database') + LOG.info('Attempting to insert metadata to database') await self._conn.execute("""INSERT INTO beacon_dataset_table (name, datasetId, description, assemblyId, createDateTime, updateDateTime, version, diff --git a/beacon_api/utils/validate_json.py b/beacon_api/utils/validate_json.py new file mode 100644 index 00000000..f89f879f --- /dev/null +++ b/beacon_api/utils/validate_json.py @@ -0,0 +1,92 @@ +"""JSON Request/Response Validation.""" + +from functools import wraps +from aiohttp import web +from .logging import LOG +from ..api.exceptions import BeaconBadRequest, BeaconServerError + +from jsonschema import Draft7Validator, validators +from jsonschema.exceptions import ValidationError + +from typing import Dict, Tuple, Callable, Any + + +async def parse_request_object(request: web.Request) -> Tuple[str, Dict]: + """Parse as JSON Object depending on the request method. + + For POST request parse the body, while for the GET request parse the query parameters. + """ + items = dict() + + if request.method == 'POST': + LOG.info('Parsed POST request body.') + items = await request.json() # we are always expecting JSON + + if request.method == 'GET': + # GET parameters are returned as strings + int_params = ['start', 'end', 'endMax', 'endMin', 'startMax', 'startMin'] + items = {k: (int(v) if k in int_params else v) for k, v in request.rel_url.query.items()} + if 'datasetIds' in items: + items['datasetIds'] = request.rel_url.query.get('datasetIds').split(',') + LOG.info('Parsed GET request parameters.') + + return request.method, items + + +def extend_with_default(validator_class: Draft7Validator) -> Draft7Validator: + """Include default values present in JSON Schema. + + Source: https://python-jsonschema.readthedocs.io/en/latest/faq/#why-doesn-t-my-schema-s-default-property-set-the-default-on-my-instance + """ + validate_properties = validator_class.VALIDATORS["properties"] + + def set_defaults(validator, properties, instance, schema): + for property, subschema in properties.items(): + if "default" in subschema: + instance.setdefault(property, subschema["default"]) + + for error in validate_properties( + validator, properties, instance, schema, + ): + # Difficult to unit test + yield error # pragma: no cover + + return validators.extend( + validator_class, {"properties": set_defaults}, + ) + + +DefaultValidatingDraft7Validator = extend_with_default(Draft7Validator) + + +def validate(schema: Dict) -> Callable[[Any], Any]: + """ + Validate against JSON schema and return errors, if any. + + Return a parsed object if there is a POST. + If there is a get do not return anything just validate. + """ + def wrapper(func): + + @wraps(func) + async def wrapped(*args): + request = args[-1] + try: + _, obj = await parse_request_object(request) + except Exception: + raise BeaconServerError("Could not properly parse the provided Request Body as JSON.") + try: + # jsonschema.validate(obj, schema) + LOG.info('Validate against JSON schema.') + DefaultValidatingDraft7Validator(schema).validate(obj) + except ValidationError as e: + if len(e.path) > 0: + LOG.error(f'Bad Request: {e.message} caused by input: {e.instance} in {e.path[0]}') + raise BeaconBadRequest(obj, request.host, f"Provided input: '{e.instance}' does not seem correct for field: '{e.path[0]}'") + else: + LOG.error(f'Bad Request: {e.message} caused by input: {e.instance}') + raise BeaconBadRequest(obj, request.host, f"Provided input: '{e.instance}' does not seem correct because: '{e.message}'") + + return await func(*args) + return wrapped + return wrapper diff --git a/beacon_api/utils/validate.py b/beacon_api/utils/validate_jwt.py similarity index 61% rename from beacon_api/utils/validate.py rename to beacon_api/utils/validate_jwt.py index e40817fc..b2f8e75d 100644 --- a/beacon_api/utils/validate.py +++ b/beacon_api/utils/validate_jwt.py @@ -1,106 +1,26 @@ -"""JSON Request/Response Validation and Token authentication.""" +"""JSON Token authentication.""" +from typing import List, Callable, Set +from ..permissions.ga4gh import check_ga4gh_token +from aiocache import cached +from aiocache.serializers import JsonSerializer +from ..api.exceptions import BeaconUnauthorised, BeaconForbidden, BeaconServerError from aiohttp import web from authlib.jose import jwt from authlib.jose.errors import MissingClaimError, InvalidClaimError, ExpiredTokenError, InvalidTokenError import re import aiohttp -import os -from functools import wraps +from os import environ from .logging import LOG -from aiocache import cached -from aiocache.serializers import JsonSerializer -from ..api.exceptions import BeaconUnauthorised, BeaconBadRequest, BeaconForbidden, BeaconServerError from ..conf import OAUTH2_CONFIG -from ..permissions.ga4gh import check_ga4gh_token -from jsonschema import Draft7Validator, validators -from jsonschema.exceptions import ValidationError - - -async def parse_request_object(request): - """Parse as JSON Object depending on the request method. - - For POST request parse the body, while for the GET request parse the query parameters. - """ - if request.method == 'POST': - LOG.info('Parsed POST request body.') - return request.method, await request.json() # we are always expecting JSON - - if request.method == 'GET': - # GET parameters are returned as strings - int_params = ['start', 'end', 'endMax', 'endMin', 'startMax', 'startMin'] - items = {k: (int(v) if k in int_params else v) for k, v in request.rel_url.query.items()} - if 'datasetIds' in items: - items['datasetIds'] = request.rel_url.query.get('datasetIds').split(',') - LOG.info('Parsed GET request parameters.') - return request.method, items - - -# TO DO if required do not set default -def extend_with_default(validator_class): - """Include default values present in JSON Schema. - - Source: https://python-jsonschema.readthedocs.io/en/latest/faq/#why-doesn-t-my-schema-s-default-property-set-the-default-on-my-instance - """ - validate_properties = validator_class.VALIDATORS["properties"] - - def set_defaults(validator, properties, instance, schema): - for property, subschema in properties.items(): - if "default" in subschema: - instance.setdefault(property, subschema["default"]) - - for error in validate_properties( - validator, properties, instance, schema, - ): - # Difficult to unit test - yield error # pragma: no cover - - return validators.extend( - validator_class, {"properties": set_defaults}, - ) - - -DefaultValidatingDraft7Validator = extend_with_default(Draft7Validator) - - -def validate(schema): - """ - Validate against JSON schema an return something. - - Return a parsed object if there is a POST. - If there is a get do not return anything just validate. - """ - def wrapper(func): - - @wraps(func) - async def wrapped(*args): - request = args[-1] - try: - _, obj = await parse_request_object(request) - except Exception: - raise BeaconServerError("Could not properly parse the provided Request Body as JSON.") - try: - # jsonschema.validate(obj, schema) - LOG.info('Validate against JSON schema.') - DefaultValidatingDraft7Validator(schema).validate(obj) - except ValidationError as e: - if len(e.path) > 0: - LOG.error(f'Bad Request: {e.message} caused by input: {e.instance} in {e.path[0]}') - raise BeaconBadRequest(obj, request.host, f"Provided input: '{e.instance}' does not seem correct for field: '{e.path[0]}'") - else: - LOG.error(f'Bad Request: {e.message} caused by input: {e.instance}') - raise BeaconBadRequest(obj, request.host, f"Provided input: '{e.instance}' does not seem correct because: '{e.message}'") - - return await func(*args) - return wrapped - return wrapper +from .validate_json import parse_request_object # This can be something that lives longer as it is unlikely to change @cached(ttl=3600, key="jwk_key", serializer=JsonSerializer()) async def get_key(): """Get OAuth2 public key and transform it to usable pem key.""" - existing_key = os.environ.get('PUBLIC_KEY', None) + existing_key = environ.get('PUBLIC_KEY', None) if existing_key is not None: return existing_key try: @@ -122,27 +42,31 @@ def token_scheme_check(token, scheme, obj, host): raise BeaconUnauthorised(obj, host, "invalid_token", 'Token cannot be empty.') # pragma: no cover -def verify_aud_claim(): +def verify_aud_claim() -> tuple: """Verify audience claim.""" - aud = [] + aud: List[str] = [] verify_aud = OAUTH2_CONFIG.verify_aud # Option to skip verification of `aud` claim if verify_aud: - aud = os.environ.get('JWT_AUD', OAUTH2_CONFIG.audience) # List of intended audiences of token + temp_aud = environ.get('JWT_AUD', OAUTH2_CONFIG.audience) # List of intended audiences of token # if verify_aud is set to True, we expect that a desired aud is then supplied. # However, if verify_aud=True and no aud is supplied, we use aud=[None] which will fail for # all tokens as a security measure. If aud=[], all tokens will pass (as is the default value). - aud = aud.split(',') if aud is not None else [None] + if temp_aud is not None: + aud = temp_aud.split(',') + else: + aud.append[None] + return verify_aud, aud -def token_auth(): +def token_auth() -> Callable: """Check if token is valid and authenticate. Decided against: https://github.com/hzlmn/aiohttp-jwt, as we need to verify token issuer and bona_fide_status. """ @web.middleware - async def token_middleware(request, handler): + async def token_middleware(request: web.Request, handler): if request.path in ['/query'] and 'Authorization' in request.headers: _, obj = await parse_request_object(request) try: @@ -182,12 +106,13 @@ async def token_middleware(request, handler): # the bona fide status is checked against ELIXIR AAI by default or the URL from config # the bona_fide_status is specific to ELIXIR Tokens # Retrieve GA4GH Passports from /userinfo and process them into dataset permissions and bona fide status - dataset_permissions, bona_fide_status = set(), False + dataset_permissions: Set[str] = set() + bona_fide_status: bool = False dataset_permissions, bona_fide_status = await check_ga4gh_token(decoded_data, token, bona_fide_status, dataset_permissions) # currently we offer module for parsing GA4GH permissions, but multiple claims and providers can be utilised # by updating the set, meaning replicating the line below with the permissions function and its associated claim # For GA4GH DURI permissions (ELIXIR Permissions API 2.0) - controlled_datasets = set() + controlled_datasets: Set[str] = set() controlled_datasets.update(dataset_permissions) all_controlled = list(controlled_datasets) if bool(controlled_datasets) else None request["token"] = {"bona_fide_status": bona_fide_status, diff --git a/deploy/test/auth_test.ini b/deploy/test/auth_test.ini index 1cf90aaf..ed34eeae 100644 --- a/deploy/test/auth_test.ini +++ b/deploy/test/auth_test.ini @@ -7,7 +7,7 @@ title=GA4GHBeacon at CSC # Version of the Beacon implementation -version=1.6.1 +version=1.7.0 # Author of this software author=CSC developers diff --git a/deploy/test/integ_test.py b/deploy/test/integ_test.py index 87faf1b8..b29bdb8f 100644 --- a/deploy/test/integ_test.py +++ b/deploy/test/integ_test.py @@ -27,7 +27,7 @@ TOKEN_EMPTY = result[1] -async def test_1(): +async def test_1() -> None: """Test the info endpoint. Info endpoint should respond with 4 datasets all in the list specified above. @@ -44,7 +44,7 @@ async def test_1(): sys.exit('Info Endpoint Error!') -async def test_2(): +async def test_2() -> None: """Test query GET endpoint. Send a query with alternateBases. Expect data to be found (200). @@ -67,7 +67,7 @@ async def test_2(): sys.exit('Query GET Endpoint Error!') -async def test_3(): +async def test_3() -> None: """Test query GET endpoint. Send a query with variantType. Expect data to be found (200). @@ -88,7 +88,7 @@ async def test_3(): sys.exit('Query GET Endpoint Error!') -async def test_4(): +async def test_4() -> None: """Test query GET endpoint. Send a query with missing required params. Expect a bad request (400). @@ -111,7 +111,7 @@ async def test_4(): sys.exit('Query GET Endpoint Error!') -async def test_5(): +async def test_5() -> None: """Test query GET endpoint. Send a query with wildcard alternateBases. Expect data to be found (200). @@ -133,7 +133,7 @@ async def test_5(): sys.exit('Query GET Endpoint Error!') -async def test_6(): +async def test_6() -> None: """Test query POST endpoint. Send a query with alternateBases. Expect data to be found (200). @@ -160,7 +160,7 @@ async def test_6(): sys.exit('Query POST Endpoint Error!') -async def test_7(): +async def test_7() -> None: """Test query POST endpoint. Send a query with variantType. Expect data to be found (200). @@ -185,7 +185,7 @@ async def test_7(): sys.exit('Query POST Endpoint Error!') -async def test_8(): +async def test_8() -> None: """Test query POST endpoint. Send a query with missing required params. Expect a bad request (400). @@ -209,7 +209,7 @@ async def test_8(): sys.exit('Query POST Endpoint Error!') -async def test_9(): +async def test_9() -> None: """Test query GET endpoint. Send a query with wildcard alternateBases. Expect no data to be found exists=false, but query was good (200). @@ -224,7 +224,7 @@ async def test_9(): assert data['exists'] is False, sys.exit('Query GET Endpoint Error!') -async def test_10(): +async def test_10() -> None: """Test query POST endpoint. Send a query targeted to a REGISTERED dataset without bona_fide_status. Expect failure (401). @@ -245,7 +245,7 @@ async def test_10(): assert resp.status == 401, 'HTTP Status code error' -async def test_11(): +async def test_11() -> None: """Test query POST endpoint. Send a query targeted to a CONTROLLED dataset without token perms. Expect failure (401). @@ -266,7 +266,7 @@ async def test_11(): assert resp.status == 401, 'HTTP Status code error' -async def test_12(): +async def test_12() -> None: """Test query POST endpoint. Send a multiquery targeting PUBLIC and CONTROLLED datasets without token perms. Expect only public data to be shown (200). @@ -286,7 +286,7 @@ async def test_12(): assert len(data['datasetAlleleResponses']) == 1, sys.exit('Should be able to retrieve only public.') -async def test_13(): +async def test_13() -> None: """Test query POST endpoint. Send a multiquery targeting PUBLIC and REGISTERED datasets with bona_fide_status. Expect data to be found (200). @@ -307,7 +307,7 @@ async def test_13(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should be able to retrieve both requested.') -async def test_14(): +async def test_14() -> None: """Test query POST endpoint. Send a multiquery targeting REGISTERED and CONTROLLED datasets with bona_fide_status and token perms. Expect data to be found (200). @@ -328,7 +328,7 @@ async def test_14(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should be able to retrieve both requested.') -async def test_15(): +async def test_15() -> None: """Test query POST endpoint. Send a query targeting CONTROLLED dataset without token perms. Expect failure (403). @@ -350,7 +350,7 @@ async def test_15(): assert resp.status == 401, 'HTTP Status code error' -async def test_16(): +async def test_16() -> None: """Test query POST endpoint. Send a query targeting REGISTERED dataset with token, but no bona fide. Expect failure (403). @@ -371,7 +371,7 @@ async def test_16(): assert resp.status == 401, 'HTTP Status code error' -async def test_17(): +async def test_17() -> None: """Test query POST endpoint. Send a query targeting two CONTROLLED dataset with token perms, having access only to one of them. Expect data to be found (200). @@ -392,7 +392,7 @@ async def test_17(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should be able to retrieve both requested.') -async def test_18(): +async def test_18() -> None: """Test query POST endpoint. Send a query with bad end parameter. Expect failure (400). @@ -412,7 +412,7 @@ async def test_18(): assert resp.status == 400, 'HTTP Status code error' -async def test_19(): +async def test_19() -> None: """Test query POST endpoint. Send a query with bad start min/max parameters. Expect failure (400). @@ -432,7 +432,7 @@ async def test_19(): assert resp.status == 400, 'HTTP Status code error' -async def test_20(): +async def test_20() -> None: """Test query POST endpoint. Send a query with bad end min/max parameters. Expect failure (400). @@ -452,7 +452,7 @@ async def test_20(): assert resp.status == 400, 'HTTP Status code error' -async def test_21(): +async def test_21() -> None: """Test query POST endpoint. Send a query for non-existing variant targeting PUBLIC and CONTROLLED datasets with token perms, using MISS. @@ -474,7 +474,7 @@ async def test_21(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should be able to retrieve only public.') -async def test_22(): +async def test_22() -> None: """Test query POST endpoint. Send a query for non-existing variant targeting CONTROLLED datasets with token perms, using MISS. @@ -496,7 +496,7 @@ async def test_22(): assert len(data['datasetAlleleResponses']) == 1, sys.exit('Should be able to retrieve only public.') -async def test_23(): +async def test_23() -> None: """Test query POST endpoint. Send a query for targeting a non-existing PUBLIC datasets, using ALL. @@ -518,7 +518,7 @@ async def test_23(): assert len(data['datasetAlleleResponses']) == 0, sys.exit('Should be able to retrieve only public.') -async def test_24(): +async def test_24() -> None: """Test query POST endpoint. Send a query for targeting one existing and one non-existing PUBLIC datasets, using ALL. @@ -540,7 +540,7 @@ async def test_24(): assert len(data['datasetAlleleResponses']) == 1, sys.exit('Should be able to retrieve only public.') -async def test_25(): +async def test_25() -> None: """Test query POST endpoint. Send a query for non-existing variant targeting three datasets, using ALL. @@ -562,7 +562,7 @@ async def test_25(): assert len(data['datasetAlleleResponses']) == 3, sys.exit('Should be able to retrieve data for all datasets.') -async def test_26(): +async def test_26() -> None: """Test query POST endpoint. Send a query for non-existing variant targeting three datasets, using MISS. @@ -584,7 +584,7 @@ async def test_26(): assert len(data['datasetAlleleResponses']) == 3, sys.exit('Should be able to retrieve missing datasets.') -async def test_27(): +async def test_27() -> None: """Test query POST endpoint. Send a query targeting three datasets, using MISS. @@ -606,7 +606,7 @@ async def test_27(): assert len(data['datasetAlleleResponses']) == 0, sys.exit('Should not be able to retrieve any datasets.') -async def test_28(): +async def test_28() -> None: """Test query POST endpoint. Test BND query when end is smaller than start, with variantType and no mateName. Expect two hits, one for each direction (200). @@ -628,7 +628,7 @@ async def test_28(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should not be able to retrieve any datasets.') -async def test_29(): +async def test_29() -> None: """Test query POST endpoint. Test BND query with mateName and no variantType. Expect two hits, one for each direction (200). @@ -649,7 +649,7 @@ async def test_29(): assert len(data['datasetAlleleResponses']) == 2, sys.exit('Should not be able to retrieve any datasets.') -async def test_30(): +async def test_30() -> None: """Test query POST endpoint. Test mateName query without variantType, where end is smaller than start. @@ -671,7 +671,7 @@ async def test_30(): assert resp.status == 400, 'HTTP Status code error' -async def test_31(): +async def test_31() -> None: """Test query POST endpoint. Test mateName query with startMin and startMax with no end params. Expect good query (200). @@ -692,7 +692,7 @@ async def test_31(): assert resp.status == 200, 'HTTP Status code error' -async def test_32(): +async def test_32() -> None: """Test the GA4GH Discovery info endpoint. Discovery endpoint should be smaller than Beacon info endpoint. diff --git a/deploy/test/mock_auth.py b/deploy/test/mock_auth.py index 6bbe9e3f..6d050cfb 100644 --- a/deploy/test/mock_auth.py +++ b/deploy/test/mock_auth.py @@ -5,11 +5,12 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend from authlib.jose import jwt, jwk +from typing import Tuple -def generate_token(): +def generate_token() -> Tuple: """Generate RSA Key pair to be used to sign token and the JWT Token itself.""" - private_key = rsa.generate_private_key(public_exponent=65537, key_size=1024, backend=default_backend()) + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) public_key = private_key.public_key().public_bytes(encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo) pem = private_key.private_bytes(encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, @@ -116,7 +117,7 @@ def generate_token(): DATA = generate_token() -async def jwk_response(request): +async def jwk_response(request: web.Request) -> web.Response: """Mock JSON Web Key server.""" keys = [DATA[0]] keys[0]['kid'] = 'rsa1' @@ -126,13 +127,13 @@ async def jwk_response(request): return web.json_response(data) -async def tokens_response(request): +async def tokens_response(request: web.Request) -> web.Response: """Serve generated tokens.""" data = [DATA[1], DATA[2]] return web.json_response(data) -async def userinfo(request): +async def userinfo(request: web.Request) -> web.Response: """Mock an authentication to ELIXIR AAI for GA4GH claims.""" if request.headers.get('Authorization').split(' ')[1] == DATA[2]: data = {} @@ -148,7 +149,7 @@ async def userinfo(request): return web.json_response(data) -def init(): +def init() -> web.Application: """Start server.""" app = web.Application() app.router.add_get('/jwk', jwk_response) diff --git a/deploy/test/run_tests.py b/deploy/test/run_tests.py index 9c2ec3ad..c08aa5a0 100644 --- a/deploy/test/run_tests.py +++ b/deploy/test/run_tests.py @@ -4,7 +4,7 @@ import asyncio -async def main(): +async def main() -> None: """Run the tests.""" LOG.debug('Start integration tests') # tests 18, 19 and 20 are also tested in the unit tests diff --git a/docs/code.rst b/docs/code.rst index b9e4aadd..903ff3e5 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -59,7 +59,8 @@ Utility Functions beacon_api.utils.logging beacon_api.utils.db_load - beacon_api.utils.validate + beacon_api.utils.validate_json + beacon_api.utils.validate_jwt beacon_api.utils.data_query ****************** diff --git a/docs/conf.py b/docs/conf.py index 6fc6b94a..56480723 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -97,7 +97,7 @@ def __getattr__(cls, name): # -- Options for HTML output ---------------------------------------------- -html_title = 'Beacon-python API' +html_title = 'beacon-python API' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. diff --git a/docs/db.rst b/docs/db.rst index 782e38c1..1ce6cd5c 100644 --- a/docs/db.rst +++ b/docs/db.rst @@ -3,7 +3,7 @@ Database ======== -We use a PostgreSQL database (recomended version 11.6) for working with beacon data. +We use a PostgreSQL database (recommended version 11.6) for working with beacon data. For more information on setting up the database consult :ref:`database-setup`. .. attention:: We recommend https://pgtune.leopard.in.ua/ for establishing PostgreSQL diff --git a/docs/docs.txt b/docs/docs.txt index 856b3f4c..50e5756a 100644 --- a/docs/docs.txt +++ b/docs/docs.txt @@ -1,2 +1,3 @@ sphinx -sphinx_rtd_theme \ No newline at end of file +sphinx_rtd_theme +aiohttp \ No newline at end of file diff --git a/docs/example.rst b/docs/example.rst index daba8566..0f639df4 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -126,7 +126,7 @@ Example Response: "createdAt": "2019-09-04T12:00:00Z", "updatedAt": "2019-09-05T05:55:18Z", "environment": "prod", - "version": "1.6.1" + "version": "1.7.0" } Query Endpoint diff --git a/docs/permissions.rst b/docs/permissions.rst index c4079126..87b78d4d 100644 --- a/docs/permissions.rst +++ b/docs/permissions.rst @@ -60,12 +60,12 @@ and retrieved as illustrated in: :language: python :lines: 248-264 -The permissions are then passed in :meth:`beacon_api.utils.validate` as illustrated below: +The permissions are then passed in :meth:`beacon_api.utils.validate_jwt` as illustrated below: -.. literalinclude:: /../beacon_api/utils/validate.py +.. literalinclude:: /../beacon_api/utils/validate_jwt.py :language: python :dedent: 16 - :lines: 180-197 + :lines: 101-123 If there is no claim for GA4GH permissions as illustrated above, they will not be added to ``controlled_datasets``. diff --git a/setup.py b/setup.py index e33b164b..03b9480e 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup +from setuptools import find_packages, setup from beacon_api import __license__, __version__, __author__, __description__ @@ -13,11 +13,10 @@ author_email='', description=__description__, long_description="", - packages=['beacon_api', 'beacon_api/utils', 'beacon_api/conf', - 'beacon_api/schemas', 'beacon_api/api', 'beacon_api/permissions', - 'beacon_api/extensions'], - # If any package contains *.json, include them: + packages=find_packages(exclude=["tests", "docs"]), + # If any package contains *.json, or config in *.ini, include them: package_data={'': ['*.json', '*.ini']}, + include_package_data=True, entry_points={ 'console_scripts': [ 'beacon=beacon_api.app:main', @@ -25,29 +24,26 @@ ] }, platforms='any', - classifiers=[ # Optional - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable + classifiers=[ 'Development Status :: 5 - Production/Stable', - # Indicate who your project is intended for 'Intended Audience :: Developers', 'Intended Audience :: Healthcare Industry', 'Intended Audience :: Information Technology', 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', 'Topic :: Scientific/Engineering :: Bio-Informatics', - # Pick your license as you wish 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', ], - install_requires=['asyncpg', 'authlib', 'aiohttp_cors', - 'jsonschema', 'gunicorn>=20.0.1', 'aiohttp'], + install_requires=['asyncpg', 'aiohttp', 'authlib', 'aiohttp_cors', + 'jsonschema', 'gunicorn>=20.0.1', + 'ujson', 'uvloop', 'aiocache', 'ujson', 'aiomcache'], extras_require={ + 'vcf': ["cyvcf2==0.10.1; python_version < '3.7'", 'numpy', + "cyvcf2; python_version >= '3.7'", 'Cython'], 'test': ['coverage==4.5.4', 'pytest<5.4', 'pytest-cov', 'coveralls', 'testfixtures', 'tox', 'flake8', 'flake8-docstrings', 'asynctest', 'aioresponses'], diff --git a/tests/coveralls.py b/tests/coveralls.py index 4de55bb1..cd0978ed 100644 --- a/tests/coveralls.py +++ b/tests/coveralls.py @@ -8,7 +8,7 @@ # Solution provided by https://stackoverflow.com/questions/32757765/conditional-commands-in-tox-tox-travis-ci-and-coveralls if __name__ == '__main__': - if 'TRAVIS' in os.environ: + if 'COVERALLS_REPO_TOKEN' in os.environ: rc = call('coveralls') sys.stdout.write("Coveralls report from TRAVIS CI.\n") # raise SystemExit(rc) diff --git a/tests/test.ini b/tests/test.ini index c88ffc9a..35f87f65 100644 --- a/tests/test.ini +++ b/tests/test.ini @@ -7,7 +7,7 @@ title=GA4GHBeacon at CSC # Version of the Beacon implementation -version=1.6.1 +version=1.7.0 # Author of this software author=CSC developers diff --git a/tests/test_app.py b/tests/test_app.py index 1937dd99..6182cafa 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -109,7 +109,7 @@ async def test_beacon_info(self): """ with asynctest.mock.patch('beacon_api.app.beacon_info', return_value={"id": "value"}): resp = await self.client.request("GET", "/") - assert 200 == resp.status + self.assertEqual(200, resp.status) @unittest_run_loop async def test_ga4gh_info(self): @@ -119,7 +119,7 @@ async def test_ga4gh_info(self): """ with asynctest.mock.patch('beacon_api.app.ga4gh_info', return_value={"id": "value"}): resp = await self.client.request("GET", "/service-info") - assert 200 == resp.status + self.assertEqual(200, resp.status) @unittest_run_loop async def test_post_info(self): @@ -128,7 +128,7 @@ async def test_post_info(self): The status should always be 405. """ resp = await self.client.request("POST", "/") - assert 405 == resp.status + self.assertEqual(405, resp.status) @unittest_run_loop async def test_post_service_info(self): @@ -137,19 +137,19 @@ async def test_post_service_info(self): The status should always be 405. """ resp = await self.client.request("POST", "/service-info") - assert 405 == resp.status + self.assertEqual(405, resp.status) @unittest_run_loop async def test_empty_get_query(self): """Test empty GET query endpoint.""" resp = await self.client.request("GET", "/query") - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_empty_post_query(self): """Test empty POST query endpoint.""" resp = await self.client.request("POST", "/query", data=json.dumps({})) - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_bad_start_post_query(self): @@ -164,7 +164,7 @@ async def test_bad_start_post_query(self): "assemblyId": "GRCh38", "includeDatasetResponses": "HIT"} resp = await self.client.request("POST", "/query", data=json.dumps(bad_start)) - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_bad_start2_post_query(self): @@ -179,7 +179,7 @@ async def test_bad_start2_post_query(self): "assemblyId": "GRCh38", "includeDatasetResponses": "HIT"} resp = await self.client.request("POST", "/query", data=json.dumps(bad_start)) - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_bad_startend_post_query(self): @@ -192,7 +192,7 @@ async def test_bad_startend_post_query(self): "assemblyId": "GRCh38", "includeDatasetResponses": "HIT"} resp = await self.client.request("POST", "/query", data=json.dumps(bad_start)) - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_bad_startminmax_post_query(self): @@ -205,7 +205,7 @@ async def test_bad_startminmax_post_query(self): "assemblyId": "GRCh38", "includeDatasetResponses": "HIT"} resp = await self.client.request("POST", "/query", data=json.dumps(bad_start)) - assert 400 == resp.status + self.assertEqual(400, resp.status) @unittest_run_loop async def test_bad_endminmax_post_query(self): @@ -218,7 +218,7 @@ async def test_bad_endminmax_post_query(self): "assemblyId": "GRCh38", "includeDatasetResponses": "HIT"} resp = await self.client.request("POST", "/query", data=json.dumps(bad_start)) - assert 400 == resp.status + self.assertEqual(400, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler') @@ -233,7 +233,7 @@ async def test_good_start_post_query(self, mock_handler, mock_object): "includeDatasetResponses": "HIT"} mock_handler.side_effect = json.dumps(good_start) resp = await self.client.request("POST", "/query", data=json.dumps(good_start)) - assert 200 == resp.status + self.assertEqual(200, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler') @@ -249,7 +249,7 @@ async def test_good_start2_post_query(self, mock_handler, mock_object): "includeDatasetResponses": "HIT"} mock_handler.side_effect = json.dumps(good_start) resp = await self.client.request("POST", "/query", data=json.dumps(good_start)) - assert 200 == resp.status + self.assertEqual(200, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler') @@ -265,7 +265,7 @@ async def test_good_start3_post_query(self, mock_handler, mock_object): "includeDatasetResponses": "HIT"} mock_handler.side_effect = json.dumps(good_start) resp = await self.client.request("POST", "/query", data=json.dumps(good_start)) - assert 200 == resp.status + self.assertEqual(200, resp.status) @unittest_run_loop async def test_unauthorized_no_token_post_query(self): @@ -273,7 +273,7 @@ async def test_unauthorized_no_token_post_query(self): resp = await self.client.request("POST", "/query", data=json.dumps(PARAMS), headers={'Authorization': "Bearer"}) - assert 401 == resp.status + self.assertEqual(401, resp.status) @unittest_run_loop async def test_unauthorized_token_post_query(self): @@ -281,7 +281,7 @@ async def test_unauthorized_token_post_query(self): resp = await self.client.request("POST", "/query", data=json.dumps(PARAMS), headers={'Authorization': f"Bearer {self.bad_token}"}) - assert 403 == resp.status + self.assertEqual(403, resp.status) @unittest_run_loop async def test_invalid_scheme_get_query(self): @@ -289,7 +289,7 @@ async def test_invalid_scheme_get_query(self): params = '?assemblyId=GRCh38&referenceName=1&start=10000&referenceBases=A&alternateBases=T&datasetIds=dataset1' resp = await self.client.request("GET", f"/query{params}", headers={'Authorization': "SMTH x"}) - assert 401 == resp.status + self.assertEqual(401, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler', side_effect=json.dumps(PARAMS)) @@ -300,13 +300,13 @@ async def test_valid_token_get_query(self, mock_handler, mock_object): resp = await self.client.request("POST", "/query", data=json.dumps(PARAMS), headers={'Authorization': f"Bearer {token}"}) - assert 200 == resp.status + self.assertEqual(200, resp.status) @unittest_run_loop async def test_bad_json_post_query(self): """Test bad json POST query endpoint.""" resp = await self.client.request("POST", "/query", data="") - assert 500 == resp.status + self.assertEqual(500, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler', side_effect=json.dumps(PARAMS)) @@ -316,7 +316,7 @@ async def test_valid_get_query(self, mock_handler, mock_object): params = '?assemblyId=GRCh38&referenceName=1&start=10000&referenceBases=A&alternateBases=T' with asynctest.mock.patch('beacon_api.app.initialize', side_effect=create_db_mock): resp = await self.client.request("GET", f"/query{params}") - assert 200 == resp.status + self.assertEqual(200, resp.status) @asynctest.mock.patch('beacon_api.app.parse_request_object', side_effect=mock_parse_request_object) @asynctest.mock.patch('beacon_api.app.query_request_handler', side_effect=json.dumps(PARAMS)) @@ -324,7 +324,7 @@ async def test_valid_get_query(self, mock_handler, mock_object): async def test_valid_post_query(self, mock_handler, mock_object): """Test valid POST query endpoint.""" resp = await self.client.request("POST", "/query", data=json.dumps(PARAMS)) - assert 200 == resp.status + self.assertEqual(200, resp.status) class AppTestCaseForbidden(AioHTTPTestCase): @@ -358,7 +358,7 @@ async def test_forbidden_token_get_query(self, mock_handler, mock_object): resp = await self.client.request("POST", "/query", data=json.dumps(PARAMS), headers={'Authorization': f"Bearer {token}"}) - assert 403 == resp.status + self.assertEqual(403, resp.status) class TestBasicFunctionsApp(asynctest.TestCase): diff --git a/tests/test_basic.py b/tests/test_basic.py index 584eaa73..1d103847 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -3,7 +3,7 @@ from beacon_api.utils.db_load import parse_arguments, init_beacon_db, main from beacon_api.conf.config import init_db_pool from beacon_api.api.query import access_resolution -from beacon_api.utils.validate import token_scheme_check, verify_aud_claim +from beacon_api.utils.validate_jwt import token_scheme_check, verify_aud_claim from beacon_api.permissions.ga4gh import get_ga4gh_controlled, get_ga4gh_bona_fide, validate_passport from beacon_api.permissions.ga4gh import check_ga4gh_token, decode_passport, get_ga4gh_permissions from .test_app import PARAMS, generate_token @@ -147,8 +147,11 @@ def test_access_resolution_base(self): request = PARAMS token = mock_token(False, [], False) host = 'localhost' - result = access_resolution(request, token, host, [1, 2], [3, 4], [5, 6]) - assert result == (['PUBLIC'], [1, 2]) + result = access_resolution(request, token, host, ["1", "2"], ["3", "4"], ["5", "6"]) + self.assertListEqual(result[0], ['PUBLIC']) + intermediate_list = result[1] + intermediate_list.sort() + self.assertListEqual(["1", "2"], intermediate_list) def test_access_resolution_no_controlled(self): """Test assumptions for access resolution for token but no controlled datasets. @@ -158,8 +161,11 @@ def test_access_resolution_no_controlled(self): request = PARAMS token = mock_token(False, [], True) host = 'localhost' - result = access_resolution(request, token, host, [1, 2], [3, 4], [5, 6]) - assert result == (['PUBLIC'], [1, 2]) + result = access_resolution(request, token, host, ["1", "2"], ["3", "4"], ["5", "6"]) + self.assertListEqual(result[0], ['PUBLIC']) + intermediate_list = result[1] + intermediate_list.sort() + self.assertListEqual(["1", "2"], intermediate_list) def test_access_resolution_registered(self): """Test assumptions for access resolution for token with just bona_fide. @@ -169,8 +175,11 @@ def test_access_resolution_registered(self): request = PARAMS token = mock_token(True, [], True) host = 'localhost' - result = access_resolution(request, token, host, [1, 2], [3, 4], [5, 6]) - assert result == (['PUBLIC', 'REGISTERED'], [1, 2, 3, 4]) + result = access_resolution(request, token, host, ["1", "2"], ["3", "4"], ["5", "6"]) + self.assertListEqual(result[0], ['PUBLIC', 'REGISTERED']) + intermediate_list = result[1] + intermediate_list.sort() + self.assertListEqual(["1", "2", "3", "4"], intermediate_list) def test_access_resolution_controlled_no_registered(self): """Test assumptions for access resolution for token and no bona_fide. @@ -178,10 +187,13 @@ def test_access_resolution_controlled_no_registered(self): It is based on the result of fetch_datasets_access function. """ request = PARAMS - token = mock_token(False, [5, 6], True) + token = mock_token(False, ["5", "6"], True) host = 'localhost' - result = access_resolution(request, token, host, [1, 2], [3, 4], [5, 6]) - assert result == (['PUBLIC', 'CONTROLLED'], [1, 2, 5, 6]) + result = access_resolution(request, token, host, ["1", "2"], ["3", "4"], ["5", "6"]) + self.assertListEqual(result[0], ['PUBLIC', 'CONTROLLED']) + intermediate_list = result[1] + intermediate_list.sort() + self.assertListEqual(["1", "2", "5", "6"], intermediate_list) def test_access_resolution_controlled_registered(self): """Test assumptions for access resolution for token and bona_fide. @@ -189,10 +201,13 @@ def test_access_resolution_controlled_registered(self): It is based on the result of fetch_datasets_access function. """ request = PARAMS - token = mock_token(True, [5, 6], True) + token = mock_token(True, ["5", "6"], True) host = 'localhost' - result = access_resolution(request, token, host, [1, 2], [3, 4], [5, 6]) - assert result == (['PUBLIC', 'REGISTERED', 'CONTROLLED'], [1, 2, 3, 4, 5, 6]) + result = access_resolution(request, token, host, ["1", "2"], ["3", "4"], ["5", "6"]) + self.assertListEqual(result[0], ['PUBLIC', 'REGISTERED', 'CONTROLLED']) + intermediate_list = result[1] + intermediate_list.sort() + self.assertListEqual(["1", "2", "3", "4", "5", "6"], intermediate_list) def test_access_resolution_bad_registered(self): """Test assumptions for access resolution for requested registered Unauthorized. @@ -203,7 +218,7 @@ def test_access_resolution_bad_registered(self): token = mock_token(False, [], False) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPUnauthorized): - access_resolution(request, token, host, [], [3], []) + access_resolution(request, token, host, [], ["3"], []) def test_access_resolution_no_registered2(self): """Test assumptions for access resolution for requested registered Forbidden. @@ -214,7 +229,7 @@ def test_access_resolution_no_registered2(self): token = mock_token(False, [], True) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPForbidden): - access_resolution(request, token, host, [], [4], []) + access_resolution(request, token, host, [], ["4"], []) def test_access_resolution_controlled_forbidden(self): """Test assumptions for access resolution for requested controlled Forbidden. @@ -225,7 +240,7 @@ def test_access_resolution_controlled_forbidden(self): token = mock_token(False, [7], True) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPForbidden): - access_resolution(request, token, host, [], [6], []) + access_resolution(request, token, host, [], ["6"], []) def test_access_resolution_controlled_unauthorized(self): """Test assumptions for access resolution for requested controlled Unauthorized. @@ -236,7 +251,7 @@ def test_access_resolution_controlled_unauthorized(self): token = mock_token(False, [], False) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPUnauthorized): - access_resolution(request, token, host, [], [5], []) + access_resolution(request, token, host, [], ["5"], []) def test_access_resolution_controlled_no_perms(self): """Test assumptions for access resolution for requested controlled Forbidden. @@ -244,10 +259,10 @@ def test_access_resolution_controlled_no_perms(self): It is based on the result of fetch_datasets_access function. """ request = PARAMS - token = mock_token(False, [7], True) + token = mock_token(False, ["7"], True) host = 'localhost' - result = access_resolution(request, token, host, [2], [6], []) - assert result == (['PUBLIC'], [2]) + result = access_resolution(request, token, host, ["2"], ["6"], []) + self.assertEqual(result, (['PUBLIC'], ["2"])) def test_access_resolution_controlled_some(self): """Test assumptions for access resolution for requested controlled some datasets. @@ -255,10 +270,10 @@ def test_access_resolution_controlled_some(self): It is based on the result of fetch_datasets_access function. """ request = PARAMS - token = mock_token(False, [5], True) + token = mock_token(False, ["5"], True) host = 'localhost' - result = access_resolution(request, token, host, [], [], [5, 6]) - assert result == (['CONTROLLED'], [5]) + result = access_resolution(request, token, host, [], [], ["5", "6"]) + self.assertEqual(result, (['CONTROLLED'], ["5"])) def test_access_resolution_controlled_no_perms_public(self): """Test assumptions for access resolution for requested controlled and public, returning public only. @@ -268,8 +283,8 @@ def test_access_resolution_controlled_no_perms_public(self): request = PARAMS token = mock_token(False, [], False) host = 'localhost' - result = access_resolution(request, token, host, [1], [], [5]) - assert result == (['PUBLIC'], [1]) + result = access_resolution(request, token, host, ["1"], [], ["5"]) + self.assertEqual(result, (['PUBLIC'], ["1"])) def test_access_resolution_controlled_no_perms_bonafide(self): """Test assumptions for access resolution for requested controlled and registered, returning registered only. @@ -279,8 +294,8 @@ def test_access_resolution_controlled_no_perms_bonafide(self): request = PARAMS token = mock_token(True, [], True) host = 'localhost' - result = access_resolution(request, token, host, [], [4], [7]) - assert result == (['REGISTERED'], [4]) + result = access_resolution(request, token, host, [], ["4"], ["7"]) + self.assertEqual(result, (['REGISTERED'], ["4"])) def test_access_resolution_controlled_never_reached(self): """Test assumptions for access resolution for requested controlled unauthorized. @@ -292,7 +307,7 @@ def test_access_resolution_controlled_never_reached(self): token = mock_token(False, None, False) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPUnauthorized): - access_resolution(request, token, host, [], [], [8]) + access_resolution(request, token, host, [], [], ["8"]) def test_access_resolution_controlled_never_reached2(self): """Test assumptions for access resolution for requested controlled forbidden. @@ -304,7 +319,7 @@ def test_access_resolution_controlled_never_reached2(self): token = mock_token(False, None, True) host = 'localhost' with self.assertRaises(aiohttp.web_exceptions.HTTPForbidden): - access_resolution(request, token, host, [], [], [8]) + access_resolution(request, token, host, [], [], ["8"]) @asynctest.mock.patch('beacon_api.permissions.ga4gh.validate_passport') async def test_ga4gh_controlled(self, m_validation): @@ -419,7 +434,7 @@ async def test_get_ga4gh_permissions(self, m_userinfo, m_decode, m_controlled, m m_decode.return_value = header, payload m_controlled.return_value = set() m_bonafide.return_value = False - dataset_permissions, bona_fide_status = await get_ga4gh_permissions('token') + dataset_permissions, bona_fide_status = await get_ga4gh_permissions({}) self.assertEqual(dataset_permissions, set()) self.assertEqual(bona_fide_status, False) # Test: permissions @@ -433,7 +448,7 @@ async def test_get_ga4gh_permissions(self, m_userinfo, m_decode, m_controlled, m m_decode.return_value = header, payload m_controlled.return_value = {'EGAD01'} m_bonafide.return_value = False - dataset_permissions, bona_fide_status = await get_ga4gh_permissions('token') + dataset_permissions, bona_fide_status = await get_ga4gh_permissions({}) self.assertEqual(dataset_permissions, {'EGAD01'}) self.assertEqual(bona_fide_status, False) # Test: bona fide @@ -447,7 +462,7 @@ async def test_get_ga4gh_permissions(self, m_userinfo, m_decode, m_controlled, m m_decode.return_value = header, payload m_controlled.return_value = set() m_bonafide.return_value = True - dataset_permissions, bona_fide_status = await get_ga4gh_permissions('token') + dataset_permissions, bona_fide_status = await get_ga4gh_permissions({}) self.assertEqual(dataset_permissions, set()) self.assertEqual(bona_fide_status, True) diff --git a/tests/test_db_load.py b/tests/test_db_load.py index cbdbea23..5cae846c 100644 --- a/tests/test_db_load.py +++ b/tests/test_db_load.py @@ -225,7 +225,7 @@ async def test_check_tables(self, db_mock): db_mock.assert_called() result = await self._db.check_tables(['DATATSET1', 'DATATSET2']) # No Missing tables - assert result == [] + self.assertEqual(result, []) @asynctest.mock.patch('beacon_api.utils.db_load.LOG') @asynctest.mock.patch('beacon_api.utils.db_load.asyncpg.connect') @@ -314,7 +314,7 @@ async def test_insert_variants(self, db_mock, mock_log): db_mock.assert_called() await self._db.insert_variants('DATASET1', ['C']) # Should assert logs - mock_log.info.mock_calls = [f'Received 1 variants for insertion to DATASET1', + mock_log.info.mock_calls = ['Received 1 variants for insertion to DATASET1', 'Insert variants into the database'] @asynctest.mock.patch('beacon_api.utils.db_load.LOG') diff --git a/tests/test_mate_name.py b/tests/test_mate_name.py index 2265a9e9..07f6ac69 100644 --- a/tests/test_mate_name.py +++ b/tests/test_mate_name.py @@ -19,11 +19,10 @@ def tearDown(self): async def test_find_fusion(self, mock_filtered): """Test find datasets.""" mock_filtered.return_value = [] - token = dict() - token["bona_fide_status"] = False - result = await find_fusion(None, 'GRCh38', None, 'Y', 'T', 'C', [], token, "NONE") + access_type = list() + result = await find_fusion(None, 'GRCh38', (), 'Y', 'T', 'C', [], access_type, "NONE") self.assertEqual(result, []) - result_miss = await find_fusion(None, 'GRCh38', None, 'Y', 'T', 'C', [], token, "MISS") + result_miss = await find_fusion(None, 'GRCh38', (), 'Y', 'T', 'C', [], access_type, "MISS") self.assertEqual(result_miss, []) async def test_fetch_fusion_dataset_call(self): diff --git a/tests/test_response.py b/tests/test_response.py index 17a626a0..842913a5 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -2,7 +2,7 @@ from beacon_api.api.query import query_request_handler import asynctest from beacon_api.schemas import load_schema -from beacon_api.utils.validate import get_key +from beacon_api.utils.validate_jwt import get_key from beacon_api.permissions.ga4gh import retrieve_user_data, get_jwk import jsonschema import json @@ -177,7 +177,7 @@ async def test_get_jwk_bad(self, mock_log): await get_jwk('http://test.csc.fi/jwk') mock_log.error.assert_called_with("Could not retrieve JWK from http://test.csc.fi/jwk") - @asynctest.mock.patch('beacon_api.utils.validate.OAUTH2_CONFIG', return_value={'server': None}) + @asynctest.mock.patch('beacon_api.utils.validate_jwt.OAUTH2_CONFIG', return_value={'server': None}) async def test_bad_get_key(self, oauth_none): """Test bad test_get_key.""" with self.assertRaises(aiohttp.web_exceptions.HTTPInternalServerError): diff --git a/tox.ini b/tox.ini index e1bb94c4..09942733 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,12 @@ [tox] -envlist = py{36,37},flake8,docs,bandit +envlist = py{36,37},flake8,docs,bandit,unit_tests,mypy skipsdist = True [flake8] ignore = E226,D203,D212,D213,D404,D100,D104 max-line-length = 160 max-complexity = 15 +exclude = .git, ./venv/, ./.tox/, ./build [testenv:docs] ; skip_install = true @@ -19,7 +20,7 @@ skip_install = true ; plain search for known vulnerable code deps = bandit -commands = bandit -r beacon_api/ -c .bandit.yml +commands = bandit -r beacon_api/ [testenv:flake8] skip_install = true @@ -29,10 +30,17 @@ deps = flake8-docstrings commands = flake8 . -[testenv] +[testenv:mypy] +skip_install = true +deps = + -rrequirements.txt + mypy +commands = mypy --ignore-missing-imports beacon_api/ + +[testenv:unit_tests] setenv = CONFIG_FILE = {toxinidir}/tests/test.ini -passenv = TRAVIS TRAVIS_* +passenv = COVERALLS_REPO_TOKEN deps = .[test] -rrequirements.txt @@ -40,8 +48,7 @@ deps = commands = py.test -x --cov=beacon_api tests/ --cov-fail-under=80 python {toxinidir}/tests/coveralls.py -[travis] -unignore_outcomes = True +[gh-actions] python = - 3.6: py36 - 3.7: py37 \ No newline at end of file + 3.6: flake8, unit_tests, docs, bandit, mypy + 3.7: flake8, unit_tests, docs, bandit, mypy \ No newline at end of file