From 9b855c97f5ceada2db191f7ab7391752a8cc8846 Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Mon, 4 Feb 2019 16:26:09 -0600 Subject: [PATCH 1/7] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9aece9ca..92575367 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ xmltodict==0.9.2 -e git+https://git@github.com/uc-cdis/cdislogging.git@master#egg=cdislogging -e git+https://git@github.com/NCI-GDC/cdisutils.git@f54e393c89939b2200dfae45c6235cbe2bae1206#egg=cdisutils -e git+https://git@github.com/uc-cdis/datadictionary.git@0.2.1#egg=gdcdictionary --e git+https://git@github.com/uc-cdis/gdcdatamodel.git@1.3.6#egg=gdcdatamodel +-e git+https://git@github.com/uc-cdis/gdcdatamodel.git@11bf581470c564ff99a4eec8e0b687907111e36f#egg=gdcdatamodel -e git+https://git@github.com/uc-cdis/indexclient.git@1.5.7#egg=indexclient -e git+https://git@github.com/NCI-GDC/python-signpostclient.git@ca686f55772e9a7f839b4506090e7d2bb0de5f15#egg=signpostclient -e git+https://git@github.com/uc-cdis/storage-client.git@0.1.7#egg=storageclient From 2eebffc6b6a72cdcfd9a7a83a27cb2a16f2ee8a4 Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Sat, 9 Feb 2019 00:51:30 -0600 Subject: [PATCH 2/7] bump pins --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 92575367..edd0b923 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,14 +23,14 @@ xmltodict==0.9.2 -e git+https://git@github.com/uc-cdis/cdis_oauth2client.git@0.1.3#egg=cdis_oauth2client -e git+https://git@github.com/uc-cdis/cdis-python-utils.git@0.2.8#egg=cdispyutils -e git+https://git@github.com/uc-cdis/datamodelutils.git@0.4.0#egg=datamodelutils --e git+https://git@github.com/NCI-GDC/psqlgraph.git@1.2.0#egg=psqlgraph +-e git+https://git@github.com/uc-cdis/psqlgraph.git@3b5ac0402857f990e93d8c32ced5264592360d27#egg=psqlgraph -e git+https://git@github.com/NCI-GDC/signpost.git@v1.1#egg=signpost # required for gdcdatamodel, not required for sheepdog -e git+https://git@github.com/uc-cdis/cdiserrors.git@0.1.1#egg=cdiserrors -e git+https://git@github.com/uc-cdis/cdislogging.git@master#egg=cdislogging -e git+https://git@github.com/NCI-GDC/cdisutils.git@f54e393c89939b2200dfae45c6235cbe2bae1206#egg=cdisutils -e git+https://git@github.com/uc-cdis/datadictionary.git@0.2.1#egg=gdcdictionary --e git+https://git@github.com/uc-cdis/gdcdatamodel.git@11bf581470c564ff99a4eec8e0b687907111e36f#egg=gdcdatamodel +-e git+https://git@github.com/uc-cdis/gdcdatamodel.git@26c3da68c29edcba027fa7ca62a4c75c1886bacc#egg=gdcdatamodel -e git+https://git@github.com/uc-cdis/indexclient.git@1.5.7#egg=indexclient -e git+https://git@github.com/NCI-GDC/python-signpostclient.git@ca686f55772e9a7f839b4506090e7d2bb0de5f15#egg=signpostclient -e git+https://git@github.com/uc-cdis/storage-client.git@0.1.7#egg=storageclient From b51af68dd88958c4df59c81a39975a97d237af4b Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Sat, 9 Feb 2019 01:56:33 -0600 Subject: [PATCH 3/7] fix(json): no pretty print --- sheepdog/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sheepdog/api.py b/sheepdog/api.py index 71e04495..f8591e3a 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -126,6 +126,9 @@ def app_init(app): app.config["USE_DBGAP"] = False app.config["IS_GDC"] = False + # Turn off for performance + app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False + # default settings app.config["AUTO_MIGRATE_DATABASE"] = app.config.get("AUTO_MIGRATE_DATABASE", True) app.config["REQUIRE_FILE_INDEX_EXISTS"] = ( From b310cb2282a67b4c0c52b2ac7d858950c22f6113 Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Sat, 9 Feb 2019 02:13:32 -0600 Subject: [PATCH 4/7] Black --- bin/setup_psqlgraph.py | 149 ++-- sheepdog/api.py | 2 +- tests/integration/datadict/conftest.py | 104 +-- .../datadict/submission/test_endpoints.py | 600 ++++++++-------- .../submission/test_endpoints.py | 661 +++++++++--------- 5 files changed, 771 insertions(+), 745 deletions(-) diff --git a/bin/setup_psqlgraph.py b/bin/setup_psqlgraph.py index 93d9c200..fe6e4eb8 100755 --- a/bin/setup_psqlgraph.py +++ b/bin/setup_psqlgraph.py @@ -5,13 +5,13 @@ from psqlgraph import create_all, Node, Edge -def try_drop_test_data(user, database, root_user='postgres', host='', root_password='' ): - print('Dropping old test data') - connect_str="postgres://{user}@{host}/postgres".format( - user=root_user, host=host) +def try_drop_test_data(user, database, root_user="postgres", host="", root_password=""): + print("Dropping old test data") + connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host) if root_password: - connect_str="postgres://{user}:{password}@{host}/postgres".format( - user=root_user, password=root_password, host=host) + connect_str = "postgres://{user}:{password}@{host}/postgres".format( + user=root_user, password=root_password, host=host + ) engine = create_engine(connect_str) @@ -21,28 +21,35 @@ def try_drop_test_data(user, database, root_user='postgres', host='', root_passw try: create_stmt = 'DROP DATABASE "{database}"'.format(database=database) conn.execute(create_stmt) - except Exception, msg: + except Exception as msg: logging.warning("Unable to drop test data:" + str(msg)) conn.close() -def setup_database(user, password, database, root_user='postgres', - host='', no_drop=False, no_user=False, root_password='' - ): +def setup_database( + user, + password, + database, + root_user="postgres", + host="", + no_drop=False, + no_user=False, + root_password="", +): """ setup the user and database """ - print('Setting up test database') + print("Setting up test database") if not no_drop: try_drop_test_data(user, database, root_user, host, root_password) - connect_str="postgres://{user}@{host}/postgres".format( - user=root_user, host=host) + connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host) if password: - connect_str="postgres://{user}:{password}@{host}/postgres".format( - user=root_user, password=root_password, host=host) + connect_str = "postgres://{user}:{password}@{host}/postgres".format( + user=root_user, password=root_password, host=host + ) engine = create_engine(connect_str) conn = engine.connect() @@ -51,23 +58,26 @@ def setup_database(user, password, database, root_user='postgres', create_stmt = 'CREATE DATABASE "{database}"'.format(database=database) try: conn.execute(create_stmt) - except Exception, msg: - logging.warn('Unable to create database: {}'.format(msg)) + except Exception as msg: + logging.warn("Unable to create database: {}".format(msg)) if not no_user: try: user_stmt = "CREATE USER {user} WITH PASSWORD '{password}'".format( - user=user, password=password) + user=user, password=password + ) conn.execute(user_stmt) - except Exception, msg: + except Exception as msg: logging.warn("Unable to add user:" + str(msg)) # User may already exist - GRANT privs on new db try: - perm_stmt = 'GRANT ALL PRIVILEGES ON DATABASE {database} to {password}'\ - ''.format(database=database, password=password) + perm_stmt = ( + "GRANT ALL PRIVILEGES ON DATABASE {database} to {password}" + "".format(database=database, password=password) + ) conn.execute(perm_stmt) conn.execute("commit") - except Exception, msg: + except Exception as msg: logging.warn("Unable to GRANT privs to user:" + str(msg)) conn.close() @@ -76,57 +86,80 @@ def create_tables(host, user, password, database): """ create a table """ - print('Creating tables in test database') + print("Creating tables in test database") - engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format( - user=user, host=host, pwd=password, db=database)) + engine = create_engine( + "postgres://{user}:{pwd}@{host}/{db}".format( + user=user, host=host, pwd=password, db=database + ) + ) create_all(engine) versioned_nodes.Base.metadata.create_all(engine) def create_indexes(host, user, password, database): - print('Creating indexes') - engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format( - user=user, host=host, pwd=password, db=database)) + print("Creating indexes") + engine = create_engine( + "postgres://{user}:{pwd}@{host}/{db}".format( + user=user, host=host, pwd=password, db=database + ) + ) index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c] for scls in Node.get_subclasses(): tablename = scls.__tablename__ - map(engine.execute, index( - tablename, [ - 'node_id', - ])) - map(engine.execute, [ - "CREATE INDEX ON {} USING gin (_sysan)".format(tablename), - "CREATE INDEX ON {} USING gin (_props)".format(tablename), - "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename), - ]) + map(engine.execute, index(tablename, ["node_id"])) + map( + engine.execute, + [ + "CREATE INDEX ON {} USING gin (_sysan)".format(tablename), + "CREATE INDEX ON {} USING gin (_props)".format(tablename), + "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename), + ], + ) for scls in Edge.get_subclasses(): - map(engine.execute, index( - scls.__tablename__, [ - 'src_id', - 'dst_id', - 'dst_id, src_id', - ])) + map( + engine.execute, + index(scls.__tablename__, ["src_id", "dst_id", "dst_id, src_id"]), + ) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--host", type=str, action="store", - default='localhost', help="psql-server host") - parser.add_argument("--user", type=str, action="store", - default='test', help="psql test user") - parser.add_argument("--password", type=str, action="store", - default='test', help="psql test password") - parser.add_argument("--database", type=str, action="store", - default='sheepdog_automated_test', help="psql test database") - parser.add_argument("--no-drop", action="store_true", - default=False, help="do not drop any data") - parser.add_argument("--no-user", action="store_true", - default=False, help="do not create user") + parser.add_argument( + "--host", type=str, action="store", default="localhost", help="psql-server host" + ) + parser.add_argument( + "--user", type=str, action="store", default="test", help="psql test user" + ) + parser.add_argument( + "--password", + type=str, + action="store", + default="test", + help="psql test password", + ) + parser.add_argument( + "--database", + type=str, + action="store", + default="sheepdog_automated_test", + help="psql test database", + ) + parser.add_argument( + "--no-drop", action="store_true", default=False, help="do not drop any data" + ) + parser.add_argument( + "--no-user", action="store_true", default=False, help="do not create user" + ) args = parser.parse_args() - setup_database(args.user, args.password, args.database, - no_drop=args.no_drop, no_user=args.no_user) + setup_database( + args.user, + args.password, + args.database, + no_drop=args.no_drop, + no_user=args.no_user, + ) create_tables(args.host, args.user, args.password, args.database) create_indexes(args.host, args.user, args.password, args.database) diff --git a/sheepdog/api.py b/sheepdog/api.py index f8591e3a..92ac549f 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -127,7 +127,7 @@ def app_init(app): app.config["IS_GDC"] = False # Turn off for performance - app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False + app.config["JSONIFY_PRETTYPRINT_REGULAR"] = False # default settings app.config["AUTO_MIGRATE_DATABASE"] = app.config.get("AUTO_MIGRATE_DATABASE", True) diff --git a/tests/integration/datadict/conftest.py b/tests/integration/datadict/conftest.py index 3130cb6e..4118df38 100644 --- a/tests/integration/datadict/conftest.py +++ b/tests/integration/datadict/conftest.py @@ -34,39 +34,41 @@ try: from importlib import reload # Python 3.4+ except ImportError: - from imp import reload # Python 3.0 - 3.3 + from imp import reload # Python 3.0 - 3.3 + def get_parent(path): print(path) - return path[0:path.rfind('/')] + return path[0 : path.rfind("/")] + -PATH_TO_SCHEMA_DIR = get_parent(os.path.abspath(os.path.join(os.path.realpath(__file__), os.pardir))) + '/datadict/schemas' +PATH_TO_SCHEMA_DIR = ( + get_parent(os.path.abspath(os.path.join(os.path.realpath(__file__), os.pardir))) + + "/datadict/schemas" +) -@pytest.fixture(scope='session') + +@pytest.fixture(scope="session") def pg_config(): - test_host = 'localhost' - test_user = 'test' - test_pass = 'test' - test_db = 'sheepdog_automated_test' - return dict( - host=test_host, - user=test_user, - password=test_pass, - database=test_db, - ) + test_host = "localhost" + test_user = "test" + test_pass = "test" + test_db = "sheepdog_automated_test" + return dict(host=test_host, user=test_user, password=test_pass, database=test_db) @pytest.fixture def require_index_exists_on(app, monkeypatch): - monkeypatch.setitem(app.config, 'REQUIRE_FILE_INDEX_EXISTS', True) + monkeypatch.setitem(app.config, "REQUIRE_FILE_INDEX_EXISTS", True) + @pytest.fixture def require_index_exists_off(app, monkeypatch): - monkeypatch.setitem(app.config, 'REQUIRE_FILE_INDEX_EXISTS', False) + monkeypatch.setitem(app.config, "REQUIRE_FILE_INDEX_EXISTS", False) def wait_for_indexd_alive(port): - url = 'http://localhost:{}/_status'.format(port) + url = "http://localhost:{}/_status".format(port) try: requests.get(url) except requests.ConnectionError: @@ -76,7 +78,7 @@ def wait_for_indexd_alive(port): def wait_for_indexd_not_alive(port): - url = 'http://localhost:{}/_status'.format(port) + url = "http://localhost:{}/_status".format(port) try: requests.get(url) except requests.ConnectionError: @@ -95,26 +97,30 @@ def app(tmpdir, request): reload(default_settings) # fresh files before running - for filename in ['auth.sq3', 'index.sq3', 'alias.sq3']: + for filename in ["auth.sq3", "index.sq3", "alias.sq3"]: if os.path.exists(filename): os.remove(filename) indexd_app = get_indexd_app() - indexd_init(*SIGNPOST['auth']) - indexd = Process(target=indexd_app.run, args=['localhost', port]) + indexd_init(*SIGNPOST["auth"]) + indexd = Process(target=indexd_app.run, args=["localhost", port]) indexd.start() wait_for_indexd_alive(port) gencode_json = tmpdir.mkdir("slicing").join("test_gencode.json") - gencode_json.write(json.dumps({ - 'a_gene': ['chr1', None, 200], - 'b_gene': ['chr1', 150, 300], - 'c_gene': ['chr1', 200, None], - 'd_gene': ['chr1', None, None], - })) + gencode_json.write( + json.dumps( + { + "a_gene": ["chr1", None, 200], + "b_gene": ["chr1", 150, 300], + "c_gene": ["chr1", 200, None], + "d_gene": ["chr1", None, None], + } + ) + ) def teardown(): - for filename in ['auth.sq3', 'index.sq3', 'alias.sq3']: + for filename in ["auth.sq3", "index.sq3", "alias.sq3"]: if os.path.exists(filename): os.remove(filename) @@ -130,9 +136,13 @@ def teardown(): _app.logger.setLevel(os.environ.get("GDC_LOG_LEVEL", "WARNING")) - _app.jwt_public_keys = {_app.config['USER_API']: { - 'key-test': utils.read_file('./integration/resources/keys/test_public_key.pem') - }} + _app.jwt_public_keys = { + _app.config["USER_API"]: { + "key-test": utils.read_file( + "./integration/resources/keys/test_public_key.pem" + ) + } + } return _app @@ -144,16 +154,16 @@ def tearDown(): with pg_driver.engine.begin() as conn: for table in models.Node().get_subclass_table_names(): if table != models.Node.__tablename__: - conn.execute('delete from {}'.format(table)) + conn.execute("delete from {}".format(table)) for table in models.Edge().get_subclass_table_names(): if table != models.Edge.__tablename__: - conn.execute('delete from {}'.format(table)) - conn.execute('delete from versioned_nodes') - conn.execute('delete from _voided_nodes') - conn.execute('delete from _voided_edges') - conn.execute('delete from transaction_snapshots') - conn.execute('delete from transaction_documents') - conn.execute('delete from transaction_logs') + conn.execute("delete from {}".format(table)) + conn.execute("delete from versioned_nodes") + conn.execute("delete from _voided_nodes") + conn.execute("delete from _voided_edges") + conn.execute("delete from transaction_snapshots") + conn.execute("delete from transaction_documents") + conn.execute("delete from transaction_logs") tearDown() request.addfinalizer(tearDown) @@ -167,25 +177,25 @@ def cgci_blgsp(client, admin): @pytest.fixture() def index_client(): - return IndexClient(SIGNPOST['host'], SIGNPOST['version'], SIGNPOST['auth']) + return IndexClient(SIGNPOST["host"], SIGNPOST["version"], SIGNPOST["auth"]) def dictionary_setup(_app): - print 'dictionary setup' - url = 's3://testurl' + print("dictionary setup") + url = "s3://testurl" session = requests.Session() adapter = requests_mock.Adapter() - session.mount('s3', adapter) - json_dict = json.load(open(PATH_TO_SCHEMA_DIR + '/dictionary.json')) - adapter.register_uri('GET', url, json=json_dict, status_code=200) + session.mount("s3", adapter) + json_dict = json.load(open(PATH_TO_SCHEMA_DIR + "/dictionary.json")) + adapter.register_uri("GET", url, json=json_dict, status_code=200) resp = session.get(url) - with patch('requests.get') as get_mocked: + with patch("requests.get") as get_mocked: get_mocked.return_value = resp datadictionary = DataDictionary(url=url) dictionary.init(datadictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd + models.init(md) validators.init(vd) - diff --git a/tests/integration/datadict/submission/test_endpoints.py b/tests/integration/datadict/submission/test_endpoints.py index 5a25fd84..01a4343c 100644 --- a/tests/integration/datadict/submission/test_endpoints.py +++ b/tests/integration/datadict/submission/test_endpoints.py @@ -23,10 +23,10 @@ #: Do we have a cache case setting and should we do it? CACHE_CASES = False -BLGSP_PATH = '/v0/submission/CGCI/BLGSP/' -BRCA_PATH = '/v0/submission/TCGA/BRCA/' +BLGSP_PATH = "/v0/submission/CGCI/BLGSP/" +BRCA_PATH = "/v0/submission/TCGA/BRCA/" -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data') +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") @contextlib.contextmanager @@ -35,7 +35,7 @@ def s3_conn(): mock.start(reset=False) conn = boto.connect_s3() yield conn - bucket = conn.get_bucket('test_submission') + bucket = conn.get_bucket("test_submission") for part in bucket.list_multipart_uploads(): part.cancel_upload() mock.stop() @@ -46,7 +46,7 @@ def wrapper(*args, **kwargs): mock = mock_s3() mock.start(reset=False) conn = boto.connect_s3() - conn.create_bucket('test_submission') + conn.create_bucket("test_submission") result = f(*args, **kwargs) mock.stop() @@ -56,27 +56,28 @@ def wrapper(*args, **kwargs): def put_cgci(client, auth=None): - path = '/v0/submission' + path = "/v0/submission" headers = auth - data = json.dumps({ - 'name': 'CGCI', 'type': 'program', - 'dbgap_accession_number': 'phs000235' - }) + data = json.dumps( + {"name": "CGCI", "type": "program", "dbgap_accession_number": "phs000235"} + ) r = client.put(path, headers=headers, data=data) return r def put_cgci_blgsp(client, auth=None): put_cgci(client, auth=auth) - path = '/v0/submission/CGCI/' + path = "/v0/submission/CGCI/" headers = auth - data = json.dumps({ - "type": "project", - "code": "BLGSP", - "dbgap_accession_number": 'phs000527', - "name": "Burkitt Lymphoma Genome Sequencing Project", - "state": "open" - }) + data = json.dumps( + { + "type": "project", + "code": "BLGSP", + "dbgap_accession_number": "phs000527", + "name": "Burkitt Lymphoma Genome Sequencing Project", + "state": "open", + } + ) r = client.put(path, headers=headers, data=data) assert r.status_code == 200, r.data del g.user @@ -85,21 +86,22 @@ def put_cgci_blgsp(client, auth=None): def put_tcga_brca(client, submitter): headers = submitter - data = json.dumps({ - 'name': 'TCGA', 'type': 'program', - 'dbgap_accession_number': 'phs000178' - }) - r = client.put('/v0/submission/', headers=headers, data=data) + data = json.dumps( + {"name": "TCGA", "type": "program", "dbgap_accession_number": "phs000178"} + ) + r = client.put("/v0/submission/", headers=headers, data=data) assert r.status_code == 200, r.data headers = submitter - data = json.dumps({ - "type": "project", - "code": "BRCA", - "name": "TEST", - "dbgap_accession_number": "phs000178", - "state": "open" - }) - r = client.put('/v0/submission/TCGA/', headers=headers, data=data) + data = json.dumps( + { + "type": "project", + "code": "BRCA", + "name": "TEST", + "dbgap_accession_number": "phs000178", + "state": "open", + } + ) + r = client.put("/v0/submission/TCGA/", headers=headers, data=data) assert r.status_code == 200, r.data del g.user return r @@ -108,22 +110,23 @@ def put_tcga_brca(client, submitter): def test_program_creation_endpoint(client, pg_driver, admin): resp = put_cgci(client, auth=admin) assert resp.status_code == 200, resp.data - print resp.data - resp = client.get('/v0/submission/') - assert resp.json['links'] == ['/v0/submission/CGCI'], resp.json + print(resp.data) + resp = client.get("/v0/submission/") + assert resp.json["links"] == ["/v0/submission/CGCI"], resp.json def test_program_creation_without_admin_token(client, pg_driver, submitter): - path = '/v0/submission/' + path = "/v0/submission/" headers = submitter - data = json.dumps({'name': 'CGCI', 'type': 'program'}) + data = json.dumps({"name": "CGCI", "type": "program"}) resp = client.put(path, headers=headers, data=data) assert resp.status_code == 403 def test_program_creation_endpoint_for_program_not_supported( - client, pg_driver, submitter): - path = '/v0/submission/abc/' + client, pg_driver, submitter +): + path = "/v0/submission/abc/" resp = client.post(path, headers=submitter) assert resp.status_code == 404 @@ -131,55 +134,56 @@ def test_program_creation_endpoint_for_program_not_supported( def test_project_creation_endpoint(client, pg_driver, admin): resp = put_cgci_blgsp(client, auth=admin) assert resp.status_code == 200 - resp = client.get('/v0/submission/CGCI/') + resp = client.get("/v0/submission/CGCI/") with pg_driver.session_scope(): assert pg_driver.nodes(md.Project).count() == 1 - n_cgci = ( - pg_driver.nodes(md.Project) - .path('programs') - .props(name='CGCI') - .count() - ) + n_cgci = pg_driver.nodes(md.Project).path("programs").props(name="CGCI").count() assert n_cgci == 1 - assert resp.json['links'] == ['/v0/submission/CGCI/BLGSP'], resp.json + assert resp.json["links"] == ["/v0/submission/CGCI/BLGSP"], resp.json def test_project_creation_without_admin_token(client, pg_driver, submitter, admin): put_cgci(client, admin) - path = '/v0/submission/CGCI/' + path = "/v0/submission/CGCI/" resp = client.put( - path, headers=submitter, data=json.dumps({ - "type": "project", - "code": "BLGSP", - "dbgap_accession_number": "phs000527", - "name": "Burkitt Lymphoma Genome Sequencing Project", - "state": "open"})) + path, + headers=submitter, + data=json.dumps( + { + "type": "project", + "code": "BLGSP", + "dbgap_accession_number": "phs000527", + "name": "Burkitt Lymphoma Genome Sequencing Project", + "state": "open", + } + ), + ) assert resp.status_code == 403 def test_put_entity_creation_valid(client, pg_driver, cgci_blgsp, submitter): headers = submitter - data = json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + data = json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ) resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data def test_unauthenticated_post(client, pg_driver, cgci_blgsp, submitter): # send garbage token - headers = {'Authorization': 'test'} - data = json.dumps({ - "type": "case", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + headers = {"Authorization": "test"} + data = json.dumps( + { + "type": "case", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ) resp = client.post(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 401 @@ -188,34 +192,36 @@ def test_unauthorized_post(client, pg_driver, cgci_blgsp, member): # token only has _member_ role in CGCI headers = member resp = client.post( - BLGSP_PATH, headers=headers, data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + BLGSP_PATH, + headers=headers, + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) assert resp.status_code == 403 def test_put_valid_entity_missing_target(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'sample.json'), 'r') as f: + with open(os.path.join(DATA_DIR, "sample.json"), "r") as f: sample = json.loads(f.read()) - sample['cases'] = {"submitter_id": "missing-case"} + sample["cases"] = {"submitter_id": "missing-case"} - r = client.put( - BLGSP_PATH, - headers=submitter, - data=json.dumps(sample) - ) + r = client.put(BLGSP_PATH, headers=submitter, data=json.dumps(sample)) - print r.data + print(r.data) assert r.status_code == 400, r.data - assert r.status_code == r.json['code'] - assert r.json['entities'][0]['errors'][0]['keys'] == ['cases'], r.json['entities'][0]['errors'] - assert r.json['entities'][0]['errors'][0]['type'] == 'INVALID_LINK' + assert r.status_code == r.json["code"] + assert r.json["entities"][0]["errors"][0]["keys"] == ["cases"], r.json["entities"][ + 0 + ]["errors"] + assert r.json["entities"][0]["errors"][0]["type"] == "INVALID_LINK" assert ( "[{'project_id': 'CGCI-BLGSP', 'submitter_id': 'missing-case'}]" - in r.json['entities'][0]['errors'][0]['message'] + in r.json["entities"][0]["errors"][0]["message"] ) @@ -223,57 +229,50 @@ def test_put_valid_entity_invalid_type(client, pg_driver, cgci_blgsp, submitter) r = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps([ - { - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "code": "BLGSP" - } - }, - { - "type": "case", - "submitter_id": "BLGSP-71-case-01", - "experiments": { - "submitter_id": 'BLGSP-71-06-00019' - } - }, - { - 'type': "demographic", - 'ethnicity': 'not reported', - 'gender': 'male', - 'race': 'asian', - 'submitter_id': 'demographic1', - 'year_of_birth': '1900', - 'year_of_death': 2000, - 'cases': { - 'submitter_id': 'BLGSP-71-case-01' - } - } - ])) + data=json.dumps( + [ + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"code": "BLGSP"}, + }, + { + "type": "case", + "submitter_id": "BLGSP-71-case-01", + "experiments": {"submitter_id": "BLGSP-71-06-00019"}, + }, + { + "type": "demographic", + "ethnicity": "not reported", + "gender": "male", + "race": "asian", + "submitter_id": "demographic1", + "year_of_birth": "1900", + "year_of_death": 2000, + "cases": {"submitter_id": "BLGSP-71-case-01"}, + }, + ] + ), + ) - print r.json + print(r.json) assert r.status_code == 400, r.data - assert r.status_code == r.json['code'] - assert (r.json['entities'][2]['errors'][0]['keys'] - == ['year_of_birth']), r.data - assert (r.json['entities'][2]['errors'][0]['type'] - == 'INVALID_VALUE'), r.data + assert r.status_code == r.json["code"] + assert r.json["entities"][2]["errors"][0]["keys"] == ["year_of_birth"], r.data + assert r.json["entities"][2]["errors"][0]["type"] == "INVALID_VALUE", r.data def test_post_example_entities(client, pg_driver, cgci_blgsp, submitter): path = BLGSP_PATH - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_sid = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_sid = json.loads(f.read())["submitter_id"] for fname in data_fnames: - with open(os.path.join(DATA_DIR, fname), 'r') as f: - resp = client.post( - path, headers=submitter, data=f.read() - ) + with open(os.path.join(DATA_DIR, fname), "r") as f: + resp = client.post(path, headers=submitter, data=f.read()) assert resp.status_code == 201, resp.data - if CACHE_CASES and fname not in ['experiment.json', 'case.json']: - case = resp.json['entities'][0]['related_cases'][0] - assert (case['submitter_id'] == case_sid), (fname, resp.data) + if CACHE_CASES and fname not in ["experiment.json", "case.json"]: + case = resp.json["entities"][0]["related_cases"][0] + assert case["submitter_id"] == case_sid, (fname, resp.data) def post_example_entities_together(client, submitter, data_fnames2=None): @@ -282,7 +281,7 @@ def post_example_entities_together(client, submitter, data_fnames2=None): path = BLGSP_PATH data = [] for fname in data_fnames2: - with open(os.path.join(DATA_DIR, fname), 'r') as f: + with open(os.path.join(DATA_DIR, fname), "r") as f: data.append(json.loads(f.read())) return client.post(path, headers=submitter, data=json.dumps(data)) @@ -291,90 +290,94 @@ def put_example_entities_together(client, headers): path = BLGSP_PATH data = [] for fname in data_fnames: - with open(os.path.join(DATA_DIR, fname), 'r') as f: + with open(os.path.join(DATA_DIR, fname), "r") as f: data.append(json.loads(f.read())) return client.put(path, headers=headers, data=json.dumps(data)) def test_post_example_entities_together(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_sid = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_sid = json.loads(f.read())["submitter_id"] resp = post_example_entities_together(client, submitter) - print resp.data + print(resp.data) assert resp.status_code == 201, resp.data if CACHE_CASES: - assert resp.json['entities'][2]['related_cases'][0]['submitter_id'] \ - == case_sid, resp.data + assert ( + resp.json["entities"][2]["related_cases"][0]["submitter_id"] == case_sid + ), resp.data @pytest.mark.skipif(not CACHE_CASES, reason="This dictionary does not cache cases") def test_related_cases(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_id = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_id = json.loads(f.read())["submitter_id"] resp = post_example_entities_together(client, submitter) assert resp.json["cases_related_to_created_entities_count"] == 1, resp.data assert resp.json["cases_related_to_updated_entities_count"] == 0, resp.data - for e in resp.json['entities']: - for c in e['related_cases']: - assert c['submitter_id'] == case_id, resp.data + for e in resp.json["entities"]: + for c in e["related_cases"]: + assert c["submitter_id"] == case_id, resp.data resp = put_example_entities_together(client, submitter) assert resp.json["cases_related_to_created_entities_count"] == 0, resp.data assert resp.json["cases_related_to_updated_entities_count"] == 1, resp.data def test_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary') - print resp.data - assert "/v0/submission/CGCI/BLGSP/_dictionary/slide" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/CGCI/BLGSP/_dictionary/case" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/CGCI/BLGSP/_dictionary/aliquot" \ - in json.loads(resp.data)['links'] + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary") + print(resp.data) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/slide" in json.loads(resp.data)["links"] + ) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/case" in json.loads(resp.data)["links"] + ) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/aliquot" + in json.loads(resp.data)["links"] + ) def test_top_level_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/_dictionary') - print resp.data - assert "/v0/submission/_dictionary/slide" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/_dictionary/case" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/_dictionary/aliquot" \ - in json.loads(resp.data)['links'] + resp = client.get("/v0/submission/_dictionary") + print(resp.data) + assert "/v0/submission/_dictionary/slide" in json.loads(resp.data)["links"] + assert "/v0/submission/_dictionary/case" in json.loads(resp.data)["links"] + assert "/v0/submission/_dictionary/aliquot" in json.loads(resp.data)["links"] def test_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary/aliquot') - assert json.loads(resp.data)['id'] == 'aliquot' + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary/aliquot") + assert json.loads(resp.data)["id"] == "aliquot" def test_top_level_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/_dictionary/aliquot') - assert json.loads(resp.data)['id'] == 'aliquot' + resp = client.get("/v0/submission/_dictionary/aliquot") + assert json.loads(resp.data)["id"] == "aliquot" def test_dictionary_get_definitions(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary/_definitions') - assert 'UUID' in resp.json + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary/_definitions") + assert "UUID" in resp.json def test_put_dry_run(client, pg_driver, cgci_blgsp, submitter): - path = '/v0/submission/CGCI/BLGSP/_dry_run/' + path = "/v0/submission/CGCI/BLGSP/_dry_run/" resp = client.put( path, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) assert resp.status_code == 200, resp.data resp_json = json.loads(resp.data) - assert resp_json['entity_error_count'] == 0 - assert resp_json['created_entity_count'] == 1 + assert resp_json["entity_error_count"] == 0 + assert resp_json["created_entity_count"] == 1 with pg_driver.session_scope(): assert not pg_driver.nodes(md.Experiment).first() @@ -384,28 +387,31 @@ def test_incorrect_project_error(client, pg_driver, cgci_blgsp, submitter, admin resp = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) resp = client.put( BRCA_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) resp_json = json.loads(resp.data) assert resp.status_code == 400 - assert resp_json['code'] == 400 - assert resp_json['entity_error_count'] == 1 - assert resp_json['created_entity_count'] == 0 - assert (resp_json['entities'][0]['errors'][0]['type'] - == 'INVALID_PERMISSIONS') + assert resp_json["code"] == 400 + assert resp_json["entity_error_count"] == 1 + assert resp_json["created_entity_count"] == 0 + assert resp_json["entities"][0]["errors"][0]["type"] == "INVALID_PERMISSIONS" def test_timestamps(client, pg_driver, cgci_blgsp, submitter): @@ -413,11 +419,13 @@ def test_timestamps(client, pg_driver, cgci_blgsp, submitter): with pg_driver.session_scope(): case = pg_driver.nodes(md.Case).first() ct = case.created_datetime - print case.props + print(case.props) assert ct is not None, case.props -def test_disallow_cross_project_references(client, pg_driver, cgci_blgsp, submitter, admin): +def test_disallow_cross_project_references( + client, pg_driver, cgci_blgsp, submitter, admin +): put_tcga_brca(client, admin) data = { "progression_or_recurrence": "unknown", @@ -434,18 +442,13 @@ def test_disallow_cross_project_references(client, pg_driver, cgci_blgsp, submit "age_at_diagnosis": 47, "vital_status": "dead", "morphology": "8255/3", - "cases": { - "submitter_id": "BLGSP-71-06-00019" - }, + "cases": {"submitter_id": "BLGSP-71-06-00019"}, "type": "diagnosis", "prior_malignancy": "no", "days_to_recurrence": -1, - "days_to_last_known_disease_status": -1 + "days_to_last_known_disease_status": -1, } - resp = client.put( - BRCA_PATH, - headers=submitter, - data=json.dumps(data)) + resp = client.put(BRCA_PATH, headers=submitter, data=json.dumps(data)) assert resp.status_code == 400, resp.data @@ -453,15 +456,17 @@ def test_delete_entity(client, pg_driver, cgci_blgsp, submitter): resp = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) assert resp.status_code == 200, resp.data - did = resp.json['entities'][0]['id'] - path = BLGSP_PATH + 'entities/' + did + did = resp.json["entities"][0]["id"] + path = BLGSP_PATH + "entities/" + did resp = client.delete(path, headers=submitter) assert resp.status_code == 200, resp.data @@ -473,12 +478,12 @@ def test_catch_internal_errors(monkeypatch, client, pg_driver, cgci_blgsp, submi """ def just_raise_exception(self): - raise Exception('test') + raise Exception("test") - monkeypatch.setattr(UploadTransaction, 'pre_validate', just_raise_exception) + monkeypatch.setattr(UploadTransaction, "pre_validate", just_raise_exception) try: r = put_example_entities_together(client, submitter) - assert len(r.json['transactional_errors']) == 1, r.data + assert len(r.json["transactional_errors"]) == 1, r.data except: raise @@ -489,23 +494,21 @@ def test_validator_error_types(client, pg_driver, cgci_blgsp, submitter): r = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "sample", - "cases": { - "submitter_id": "BLGSP-71-06-00019" - }, - "is_ffpe": "maybe", - "sample_type": "Blood Derived Normal", - "submitter_id": "BLGSP-71-06-00019", - "longest_dimension": -1.0 - })) - errors = { - e['keys'][0]: e['type'] - for e in r.json['entities'][0]['errors'] - } + data=json.dumps( + { + "type": "sample", + "cases": {"submitter_id": "BLGSP-71-06-00019"}, + "is_ffpe": "maybe", + "sample_type": "Blood Derived Normal", + "submitter_id": "BLGSP-71-06-00019", + "longest_dimension": -1.0, + } + ), + ) + errors = {e["keys"][0]: e["type"] for e in r.json["entities"][0]["errors"]} assert r.status_code == 400, r.data - assert errors['is_ffpe'] == 'INVALID_VALUE' - assert errors['longest_dimension'] == 'INVALID_VALUE' + assert errors["is_ffpe"] == "INVALID_VALUE" + assert errors["longest_dimension"] == "INVALID_VALUE" def test_invalid_json(client, pg_driver, cgci_blgsp, submitter): @@ -515,21 +518,21 @@ def test_invalid_json(client, pg_driver, cgci_blgsp, submitter): data="""{ "key1": "valid value", "key2": not a string, -}""") - print resp.data +}""", + ) + print(resp.data) assert resp.status_code == 400 - assert 'Expecting value' in resp.json['message'] + assert "Expecting value" in resp.json["message"] + def test_get_entity_by_id(client, pg_driver, cgci_blgsp, submitter): post_example_entities_together(client, submitter) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = '/v0/submission/CGCI/BLGSP/entities/{case_id}'.format(case_id=case_id) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/entities/{case_id}".format(case_id=case_id) + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.json['entities'][0]['properties']['id'] == case_id, r.data + assert r.json["entities"][0]["properties"]["id"] == case_id, r.data def test_invalid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter): @@ -537,31 +540,37 @@ def test_invalid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitte Test that submitting an invalid data file doesn't create an index and an alias. """ + def fail_index_test(_): - raise AssertionError('IndexClient tried to create index or alias') + raise AssertionError("IndexClient tried to create index or alias") # Since the IndexClient should never be called to register anything if the # file is invalid, change the ``create`` and ``create_alias`` methods to # raise an error. monkeypatch.setattr( - UploadTransaction, 'signpost.create', fail_index_test, raising=False + UploadTransaction, "signpost.create", fail_index_test, raising=False ) monkeypatch.setattr( - UploadTransaction, 'signpost.create_alias', fail_index_test, - raising=False + UploadTransaction, "signpost.create_alias", fail_index_test, raising=False ) # Attempt to post the invalid entities. - test_fnames = ( - data_fnames - + ['read_group.json', 'submitted_unaligned_reads_invalid.json'] - ) - resp = post_example_entities_together( - client, submitter, data_fnames2=test_fnames - ) + test_fnames = data_fnames + [ + "read_group.json", + "submitted_unaligned_reads_invalid.json", + ] + resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) print(resp) -def test_valid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter, index_client, require_index_exists_off): +def test_valid_file_index( + monkeypatch, + client, + pg_driver, + cgci_blgsp, + submitter, + index_client, + require_index_exists_off, +): """ Test that submitting a valid data file creates an index and an alias. """ @@ -570,43 +579,34 @@ def test_valid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter, # called. # Attempt to post the valid entities. - test_fnames = ( - data_fnames - + ['read_group.json', 'submitted_unaligned_reads.json'] - ) - resp = post_example_entities_together( - client, submitter, data_fnames2=test_fnames - ) + test_fnames = data_fnames + ["read_group.json", "submitted_unaligned_reads.json"] + resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) assert resp.status_code == 201, resp.data # this is a node that will have an indexd entry sur_entity = None - for entity in resp.json['entities']: - if entity['type'] == 'submitted_unaligned_reads': + for entity in resp.json["entities"]: + if entity["type"] == "submitted_unaligned_reads": sur_entity = entity - assert sur_entity, 'No submitted_unaligned_reads entity created' - assert index_client.get(sur_entity['id']), 'No indexd document created' + assert sur_entity, "No submitted_unaligned_reads entity created" + assert index_client.get(sur_entity["id"]), "No indexd document created" def test_export_entity_by_id(client, pg_driver, cgci_blgsp, submitter): post_example_entities_together(client, submitter) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = '/v0/submission/CGCI/BLGSP/export/?ids={case_id}'.format(case_id=case_id) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?ids={case_id}".format(case_id=case_id) + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') + assert r.headers["Content-Disposition"].endswith("tsv") - path += '&format=json' - r = client.get( - path, - headers=submitter) + path += "&format=json" + r = client.get(path, headers=submitter) data = r.json assert len(data) == 1 - assert data[0]['id'] == case_id + assert data[0]["id"] == case_id def test_export_all_node_types(client, pg_driver, cgci_blgsp, submitter): @@ -615,16 +615,14 @@ def test_export_all_node_types(client, pg_driver, cgci_blgsp, submitter): case = pg_driver.nodes(md.Case).first() new_case = md.Case(str(uuid.uuid4())) new_case.props = case.props - new_case.submitter_id = 'case-2' + new_case.submitter_id = "case-2" s.add(new_case) case_count = pg_driver.nodes(md.Case).count() - path = '/v0/submission/CGCI/BLGSP/export/?node_label=case' - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?node_label=case" + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') - assert len(r.data.strip().split('\n')) == case_count + 1 + assert r.headers["Content-Disposition"].endswith("tsv") + assert len(r.data.strip().split("\n")) == case_count + 1 def test_export_all_node_types_json(client, pg_driver, cgci_blgsp, submitter): @@ -633,15 +631,13 @@ def test_export_all_node_types_json(client, pg_driver, cgci_blgsp, submitter): case = pg_driver.nodes(md.Case).first() new_case = md.Case(str(uuid.uuid4())) new_case.props = case.props - new_case.submitter_id = 'case-2' + new_case.submitter_id = "case-2" s.add(new_case) case_count = pg_driver.nodes(md.Case).count() - path = '/v0/submission/CGCI/BLGSP/export/?node_label=case&format=json' - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?node_label=case&format=json" + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('json') + assert r.headers["Content-Disposition"].endswith("json") js_data = json.loads(r.data) assert len(js_data["data"]) == case_count @@ -650,50 +646,42 @@ def test_export_all_node_types_json(client, pg_driver, cgci_blgsp, submitter): def test_submit_export_encoding(client, pg_driver, cgci_blgsp, submitter): # submit metadata containing non-ascii characters headers = submitter - data = json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-submitter-ü", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + data = json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-submitter-ü", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ) resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data - node_id = resp.json['entities'][0]['id'] + node_id = resp.json["entities"][0]["id"] # TSV single node export - path = '/v0/submission/CGCI/BLGSP/export/?ids={}'.format(node_id) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?ids={}".format(node_id) + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') + assert r.headers["Content-Disposition"].endswith("tsv") tsv_output = csv.DictReader(StringIO(r.data), delimiter="\t") row = next(tsv_output) - assert row['submitter_id'] == 'BLGSP-submitter-ü' + assert row["submitter_id"] == "BLGSP-submitter-ü" # JSON single node export - path += '&format=json' - r = client.get( - path, - headers=submitter) + path += "&format=json" + r = client.get(path, headers=submitter) assert len(r.json) == 1 # TSV multiple node export - path = '/v0/submission/CGCI/BLGSP/export/?node_label=experiment' - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?node_label=experiment" + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') + assert r.headers["Content-Disposition"].endswith("tsv") tsv_output = csv.DictReader(StringIO(r.data), delimiter="\t") row = next(tsv_output) - assert row['submitter_id'] == 'BLGSP-submitter-ü' + assert row["submitter_id"] == "BLGSP-submitter-ü" # JSON multiple node export - path += '&format=json' - r = client.get( - path, - headers=submitter) - assert len(r.json) == 1 \ No newline at end of file + path += "&format=json" + r = client.get(path, headers=submitter) + assert len(r.json) == 1 diff --git a/tests/integration/datadictwithobjid/submission/test_endpoints.py b/tests/integration/datadictwithobjid/submission/test_endpoints.py index ee9f1d61..8940706c 100644 --- a/tests/integration/datadictwithobjid/submission/test_endpoints.py +++ b/tests/integration/datadictwithobjid/submission/test_endpoints.py @@ -20,10 +20,10 @@ #: Do we have a cache case setting and should we do it? CACHE_CASES = False -BLGSP_PATH = '/v0/submission/CGCI/BLGSP/' -BRCA_PATH = '/v0/submission/TCGA/BRCA/' +BLGSP_PATH = "/v0/submission/CGCI/BLGSP/" +BRCA_PATH = "/v0/submission/TCGA/BRCA/" -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data') +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") @contextlib.contextmanager @@ -32,7 +32,7 @@ def s3_conn(): mock.start(reset=False) conn = boto.connect_s3() yield conn - bucket = conn.get_bucket('test_submission') + bucket = conn.get_bucket("test_submission") for part in bucket.list_multipart_uploads(): part.cancel_upload() mock.stop() @@ -43,7 +43,7 @@ def wrapper(*args, **kwargs): mock = mock_s3() mock.start(reset=False) conn = boto.connect_s3() - conn.create_bucket('test_submission') + conn.create_bucket("test_submission") result = f(*args, **kwargs) mock.stop() @@ -53,37 +53,38 @@ def wrapper(*args, **kwargs): def put_cgci(client, auth=None): - path = '/v0/submission' + path = "/v0/submission" headers = auth - data = json.dumps({ - 'name': 'CGCI', 'type': 'program', - 'dbgap_accession_number': 'phs000235' - }) + data = json.dumps( + {"name": "CGCI", "type": "program", "dbgap_accession_number": "phs000235"} + ) r = client.put(path, headers=headers, data=data) return r + def put_cgci2(client, auth=None): - path = '/v0/submission' + path = "/v0/submission" headers = auth - data = json.dumps({ - 'name': 'CGCI2', 'type': 'program', - 'dbgap_accession_number': 'phs0002352' - }) + data = json.dumps( + {"name": "CGCI2", "type": "program", "dbgap_accession_number": "phs0002352"} + ) r = client.put(path, headers=headers, data=data) return r def put_cgci_blgsp(client, auth=None): put_cgci(client, auth=auth) - path = '/v0/submission/CGCI/' + path = "/v0/submission/CGCI/" headers = auth - data = json.dumps({ - "type": "project", - "code": "BLGSP", - "dbgap_accession_number": 'phs000527', - "name": "Burkitt Lymphoma Genome Sequencing Project", - "state": "open" - }) + data = json.dumps( + { + "type": "project", + "code": "BLGSP", + "dbgap_accession_number": "phs000527", + "name": "Burkitt Lymphoma Genome Sequencing Project", + "state": "open", + } + ) r = client.put(path, headers=headers, data=data) assert r.status_code == 200, r.data del g.user @@ -92,21 +93,22 @@ def put_cgci_blgsp(client, auth=None): def put_tcga_brca(client, submitter): headers = submitter - data = json.dumps({ - 'name': 'TCGA', 'type': 'program', - 'dbgap_accession_number': 'phs000178' - }) - r = client.put('/v0/submission/', headers=headers, data=data) + data = json.dumps( + {"name": "TCGA", "type": "program", "dbgap_accession_number": "phs000178"} + ) + r = client.put("/v0/submission/", headers=headers, data=data) assert r.status_code == 200, r.data headers = submitter - data = json.dumps({ - "type": "project", - "code": "BRCA", - "name": "TEST", - "dbgap_accession_number": "phs000178", - "state": "open" - }) - r = client.put('/v0/submission/TCGA/', headers=headers, data=data) + data = json.dumps( + { + "type": "project", + "code": "BRCA", + "name": "TEST", + "dbgap_accession_number": "phs000178", + "state": "open", + } + ) + r = client.put("/v0/submission/TCGA/", headers=headers, data=data) assert r.status_code == 200, r.data del g.user return r @@ -115,22 +117,23 @@ def put_tcga_brca(client, submitter): def test_program_creation_endpoint(client, pg_driver, admin): resp = put_cgci(client, auth=admin) assert resp.status_code == 200, resp.data - print resp.data - resp = client.get('/v0/submission/') - assert resp.json['links'] == ['/v0/submission/CGCI'], resp.json + print(resp.data) + resp = client.get("/v0/submission/") + assert resp.json["links"] == ["/v0/submission/CGCI"], resp.json def test_program_creation_without_admin_token(client, pg_driver, submitter): - path = '/v0/submission/' + path = "/v0/submission/" headers = submitter - data = json.dumps({'name': 'CGCI', 'type': 'program'}) + data = json.dumps({"name": "CGCI", "type": "program"}) resp = client.put(path, headers=headers, data=data) assert resp.status_code == 403 def test_program_creation_endpoint_for_program_not_supported( - client, pg_driver, submitter): - path = '/v0/submission/abc/' + client, pg_driver, submitter +): + path = "/v0/submission/abc/" resp = client.post(path, headers=submitter) assert resp.status_code == 404 @@ -139,72 +142,81 @@ def test_project_creation_endpoint(client, pg_driver, admin): resp = put_cgci_blgsp(client, auth=admin) assert resp.status_code == 200 - resp = client.get('/v0/submission/CGCI/') + resp = client.get("/v0/submission/CGCI/") with pg_driver.session_scope(): assert pg_driver.nodes(md.Project).count() == 1 - n_cgci = ( - pg_driver.nodes(md.Project) - .path('programs') - .props(name='CGCI') - .count() - ) + n_cgci = pg_driver.nodes(md.Project).path("programs").props(name="CGCI").count() assert n_cgci == 1 - assert resp.json['links'] == ['/v0/submission/CGCI/BLGSP'], resp.json + assert resp.json["links"] == ["/v0/submission/CGCI/BLGSP"], resp.json def test_project_creation_without_admin_token(client, pg_driver, submitter, admin): put_cgci(client, admin) - path = '/v0/submission/CGCI/' + path = "/v0/submission/CGCI/" resp = client.put( - path, headers=submitter, data=json.dumps({ - "type": "project", - "code": "BLGSP", - "dbgap_accession_number": "phs000527", - "name": "Burkitt Lymphoma Genome Sequencing Project", - "state": "open"})) + path, + headers=submitter, + data=json.dumps( + { + "type": "project", + "code": "BLGSP", + "dbgap_accession_number": "phs000527", + "name": "Burkitt Lymphoma Genome Sequencing Project", + "state": "open", + } + ), + ) assert resp.status_code == 403 -def test_project_creation_invalid_due_to_registed_project_name(client, pg_driver, admin): +def test_project_creation_invalid_due_to_registed_project_name( + client, pg_driver, admin +): resp = put_cgci_blgsp(client, auth=admin) assert resp.status_code == 200 resp = put_cgci2(client, auth=admin) assert resp.status_code == 200 - path = '/v0/submission/CGCI2/' + path = "/v0/submission/CGCI2/" resp = client.put( - path, headers=admin, data=json.dumps({ - "type": "project", - "code": "BLGSP", - "dbgap_accession_number": "phs000527", - "name": "Burkitt Lymphoma Genome Sequencing Project", - "state": "open"})) + path, + headers=admin, + data=json.dumps( + { + "type": "project", + "code": "BLGSP", + "dbgap_accession_number": "phs000527", + "name": "Burkitt Lymphoma Genome Sequencing Project", + "state": "open", + } + ), + ) assert resp.status_code == 400 def test_put_entity_creation_valid(client, pg_driver, cgci_blgsp, submitter): headers = submitter - data = json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + data = json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ) resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data def test_unauthenticated_post(client, pg_driver, cgci_blgsp, submitter): # garbage token - headers = {'Authorization': 'test'} - data = json.dumps({ - "type": "case", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + headers = {"Authorization": "test"} + data = json.dumps( + { + "type": "case", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ) resp = client.post(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 401 @@ -213,36 +225,36 @@ def test_unauthorized_post(client, pg_driver, cgci_blgsp, member): # token only has _member_ role in CGCI headers = member resp = client.post( - BLGSP_PATH, headers=headers, data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" + BLGSP_PATH, + headers=headers, + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, } - }) + ), ) assert resp.status_code == 403 def test_put_valid_entity_missing_target(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'sample.json'), 'r') as f: + with open(os.path.join(DATA_DIR, "sample.json"), "r") as f: sample = json.loads(f.read()) - sample['cases'] = {"submitter_id": "missing-case"} + sample["cases"] = {"submitter_id": "missing-case"} - r = client.put( - BLGSP_PATH, - headers=submitter, - data=json.dumps(sample) - ) + r = client.put(BLGSP_PATH, headers=submitter, data=json.dumps(sample)) - print r.data + print(r.data) assert r.status_code == 400, r.data - assert r.status_code == r.json['code'] - assert r.json['entities'][0]['errors'][0]['keys'] == ['cases'], r.json['entities'][0]['errors'] - assert r.json['entities'][0]['errors'][0]['type'] == 'INVALID_LINK' + assert r.status_code == r.json["code"] + assert r.json["entities"][0]["errors"][0]["keys"] == ["cases"], r.json["entities"][ + 0 + ]["errors"] + assert r.json["entities"][0]["errors"][0]["type"] == "INVALID_LINK" assert ( "[{'project_id': 'CGCI-BLGSP', 'submitter_id': 'missing-case'}]" - in r.json['entities'][0]['errors'][0]['message'] + in r.json["entities"][0]["errors"][0]["message"] ) @@ -250,57 +262,50 @@ def test_put_valid_entity_invalid_type(client, pg_driver, cgci_blgsp, submitter) r = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps([ - { - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "code": "BLGSP" - } - }, - { - "type": "case", - "submitter_id": "BLGSP-71-case-01", - "experiments": { - "submitter_id": 'BLGSP-71-06-00019' - } - }, - { - 'type': "demographic", - 'ethnicity': 'not reported', - 'gender': 'male', - 'race': 'asian', - 'submitter_id': 'demographic1', - 'year_of_birth': '1900', - 'year_of_death': 2000, - 'cases': { - 'submitter_id': 'BLGSP-71-case-01' - } - } - ])) + data=json.dumps( + [ + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"code": "BLGSP"}, + }, + { + "type": "case", + "submitter_id": "BLGSP-71-case-01", + "experiments": {"submitter_id": "BLGSP-71-06-00019"}, + }, + { + "type": "demographic", + "ethnicity": "not reported", + "gender": "male", + "race": "asian", + "submitter_id": "demographic1", + "year_of_birth": "1900", + "year_of_death": 2000, + "cases": {"submitter_id": "BLGSP-71-case-01"}, + }, + ] + ), + ) - print r.json + print(r.json) assert r.status_code == 400, r.data - assert r.status_code == r.json['code'] - assert (r.json['entities'][2]['errors'][0]['keys'] - == ['year_of_birth']), r.data - assert (r.json['entities'][2]['errors'][0]['type'] - == 'INVALID_VALUE'), r.data + assert r.status_code == r.json["code"] + assert r.json["entities"][2]["errors"][0]["keys"] == ["year_of_birth"], r.data + assert r.json["entities"][2]["errors"][0]["type"] == "INVALID_VALUE", r.data def test_post_example_entities(client, pg_driver, cgci_blgsp, submitter): path = BLGSP_PATH - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_sid = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_sid = json.loads(f.read())["submitter_id"] for fname in data_fnames: - with open(os.path.join(DATA_DIR, fname), 'r') as f: - resp = client.post( - path, headers=submitter, data=f.read() - ) + with open(os.path.join(DATA_DIR, fname), "r") as f: + resp = client.post(path, headers=submitter, data=f.read()) assert resp.status_code == 201, resp.data - if CACHE_CASES and fname not in ['experiment.json', 'case.json']: - case = resp.json['entities'][0]['related_cases'][0] - assert (case['submitter_id'] == case_sid), (fname, resp.data) + if CACHE_CASES and fname not in ["experiment.json", "case.json"]: + case = resp.json["entities"][0]["related_cases"][0] + assert case["submitter_id"] == case_sid, (fname, resp.data) def post_example_entities_together(client, submitter, data_fnames2=None): @@ -309,7 +314,7 @@ def post_example_entities_together(client, submitter, data_fnames2=None): path = BLGSP_PATH data = [] for fname in data_fnames2: - with open(os.path.join(DATA_DIR, fname), 'r') as f: + with open(os.path.join(DATA_DIR, fname), "r") as f: data.append(json.loads(f.read())) return client.post(path, headers=submitter, data=json.dumps(data)) @@ -318,90 +323,94 @@ def put_example_entities_together(client, headers): path = BLGSP_PATH data = [] for fname in data_fnames: - with open(os.path.join(DATA_DIR, fname), 'r') as f: + with open(os.path.join(DATA_DIR, fname), "r") as f: data.append(json.loads(f.read())) return client.put(path, headers=headers, data=json.dumps(data)) def test_post_example_entities_together(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_sid = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_sid = json.loads(f.read())["submitter_id"] resp = post_example_entities_together(client, submitter) - print resp.data + print(resp.data) assert resp.status_code == 201, resp.data if CACHE_CASES: - assert resp.json['entities'][2]['related_cases'][0]['submitter_id'] \ - == case_sid, resp.data + assert ( + resp.json["entities"][2]["related_cases"][0]["submitter_id"] == case_sid + ), resp.data @pytest.mark.skipif(not CACHE_CASES, reason="This dictionary does not cache cases") def test_related_cases(client, pg_driver, cgci_blgsp, submitter): - with open(os.path.join(DATA_DIR, 'case.json'), 'r') as f: - case_id = json.loads(f.read())['submitter_id'] + with open(os.path.join(DATA_DIR, "case.json"), "r") as f: + case_id = json.loads(f.read())["submitter_id"] resp = post_example_entities_together(client, submitter) assert resp.json["cases_related_to_created_entities_count"] == 1, resp.data assert resp.json["cases_related_to_updated_entities_count"] == 0, resp.data - for e in resp.json['entities']: - for c in e['related_cases']: - assert c['submitter_id'] == case_id, resp.data + for e in resp.json["entities"]: + for c in e["related_cases"]: + assert c["submitter_id"] == case_id, resp.data resp = put_example_entities_together(client, submitter) assert resp.json["cases_related_to_created_entities_count"] == 0, resp.data assert resp.json["cases_related_to_updated_entities_count"] == 1, resp.data def test_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary') - print resp.data - assert "/v0/submission/CGCI/BLGSP/_dictionary/slide" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/CGCI/BLGSP/_dictionary/case" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/CGCI/BLGSP/_dictionary/aliquot" \ - in json.loads(resp.data)['links'] + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary") + print(resp.data) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/slide" in json.loads(resp.data)["links"] + ) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/case" in json.loads(resp.data)["links"] + ) + assert ( + "/v0/submission/CGCI/BLGSP/_dictionary/aliquot" + in json.loads(resp.data)["links"] + ) def test_top_level_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/_dictionary') - print resp.data - assert "/v0/submission/_dictionary/slide" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/_dictionary/case" \ - in json.loads(resp.data)['links'] - assert "/v0/submission/_dictionary/aliquot" \ - in json.loads(resp.data)['links'] + resp = client.get("/v0/submission/_dictionary") + print(resp.data) + assert "/v0/submission/_dictionary/slide" in json.loads(resp.data)["links"] + assert "/v0/submission/_dictionary/case" in json.loads(resp.data)["links"] + assert "/v0/submission/_dictionary/aliquot" in json.loads(resp.data)["links"] def test_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary/aliquot') - assert json.loads(resp.data)['id'] == 'aliquot' + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary/aliquot") + assert json.loads(resp.data)["id"] == "aliquot" def test_top_level_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/_dictionary/aliquot') - assert json.loads(resp.data)['id'] == 'aliquot' + resp = client.get("/v0/submission/_dictionary/aliquot") + assert json.loads(resp.data)["id"] == "aliquot" def test_dictionary_get_definitions(client, pg_driver, cgci_blgsp, submitter): - resp = client.get('/v0/submission/CGCI/BLGSP/_dictionary/_definitions') - assert 'UUID' in resp.json + resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary/_definitions") + assert "UUID" in resp.json def test_put_dry_run(client, pg_driver, cgci_blgsp, submitter): - path = '/v0/submission/CGCI/BLGSP/_dry_run/' + path = "/v0/submission/CGCI/BLGSP/_dry_run/" resp = client.put( path, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) assert resp.status_code == 200, resp.data resp_json = json.loads(resp.data) - assert resp_json['entity_error_count'] == 0 - assert resp_json['created_entity_count'] == 1 + assert resp_json["entity_error_count"] == 0 + assert resp_json["created_entity_count"] == 1 with pg_driver.session_scope(): assert not pg_driver.nodes(md.Experiment).first() @@ -411,28 +420,31 @@ def test_incorrect_project_error(client, pg_driver, cgci_blgsp, submitter, admin resp = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) resp = client.put( BRCA_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) resp_json = json.loads(resp.data) assert resp.status_code == 400 - assert resp_json['code'] == 400 - assert resp_json['entity_error_count'] == 1 - assert resp_json['created_entity_count'] == 0 - assert (resp_json['entities'][0]['errors'][0]['type'] - == 'INVALID_PERMISSIONS') + assert resp_json["code"] == 400 + assert resp_json["entity_error_count"] == 1 + assert resp_json["created_entity_count"] == 0 + assert resp_json["entities"][0]["errors"][0]["type"] == "INVALID_PERMISSIONS" def test_timestamps(client, pg_driver, cgci_blgsp, submitter): @@ -440,11 +452,13 @@ def test_timestamps(client, pg_driver, cgci_blgsp, submitter): with pg_driver.session_scope(): case = pg_driver.nodes(md.Case).first() ct = case.created_datetime - print case.props + print(case.props) assert ct is not None, case.props -def test_disallow_cross_project_references(client, pg_driver, cgci_blgsp, submitter, admin): +def test_disallow_cross_project_references( + client, pg_driver, cgci_blgsp, submitter, admin +): put_tcga_brca(client, admin) data = { "progression_or_recurrence": "unknown", @@ -461,18 +475,13 @@ def test_disallow_cross_project_references(client, pg_driver, cgci_blgsp, submit "age_at_diagnosis": 47, "vital_status": "dead", "morphology": "8255/3", - "cases": { - "submitter_id": "BLGSP-71-06-00019" - }, + "cases": {"submitter_id": "BLGSP-71-06-00019"}, "type": "diagnosis", "prior_malignancy": "no", "days_to_recurrence": -1, - "days_to_last_known_disease_status": -1 + "days_to_last_known_disease_status": -1, } - resp = client.put( - BRCA_PATH, - headers=submitter, - data=json.dumps(data)) + resp = client.put(BRCA_PATH, headers=submitter, data=json.dumps(data)) assert resp.status_code == 400, resp.data @@ -480,15 +489,17 @@ def test_delete_entity(client, pg_driver, cgci_blgsp, submitter): resp = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) assert resp.status_code == 200, resp.data - did = resp.json['entities'][0]['id'] - path = BLGSP_PATH + 'entities/' + did + did = resp.json["entities"][0]["id"] + path = BLGSP_PATH + "entities/" + did resp = client.delete(path, headers=submitter) assert resp.status_code == 200, resp.data @@ -500,12 +511,12 @@ def test_catch_internal_errors(monkeypatch, client, pg_driver, cgci_blgsp, submi """ def just_raise_exception(self): - raise Exception('test') + raise Exception("test") - monkeypatch.setattr(UploadTransaction, 'pre_validate', just_raise_exception) + monkeypatch.setattr(UploadTransaction, "pre_validate", just_raise_exception) try: r = put_example_entities_together(client, submitter) - assert len(r.json['transactional_errors']) == 1, r.data + assert len(r.json["transactional_errors"]) == 1, r.data except: raise @@ -516,23 +527,21 @@ def test_validator_error_types(client, pg_driver, cgci_blgsp, submitter): r = client.put( BLGSP_PATH, headers=submitter, - data=json.dumps({ - "type": "sample", - "cases": { - "submitter_id": "BLGSP-71-06-00019" - }, - "is_ffpe": "maybe", - "sample_type": "Blood Derived Normal", - "submitter_id": "BLGSP-71-06-00019", - "longest_dimension": -1.0 - })) - errors = { - e['keys'][0]: e['type'] - for e in r.json['entities'][0]['errors'] - } + data=json.dumps( + { + "type": "sample", + "cases": {"submitter_id": "BLGSP-71-06-00019"}, + "is_ffpe": "maybe", + "sample_type": "Blood Derived Normal", + "submitter_id": "BLGSP-71-06-00019", + "longest_dimension": -1.0, + } + ), + ) + errors = {e["keys"][0]: e["type"] for e in r.json["entities"][0]["errors"]} assert r.status_code == 400, r.data - assert errors['is_ffpe'] == 'INVALID_VALUE' - assert errors['longest_dimension'] == 'INVALID_VALUE' + assert errors["is_ffpe"] == "INVALID_VALUE" + assert errors["longest_dimension"] == "INVALID_VALUE" def test_invalid_json(client, pg_driver, cgci_blgsp, submitter): @@ -542,22 +551,21 @@ def test_invalid_json(client, pg_driver, cgci_blgsp, submitter): data="""{ "key1": "valid value", "key2": not a string, -}""") - print resp.data +}""", + ) + print(resp.data) assert resp.status_code == 400 - assert 'Expecting value' in resp.json['message'] + assert "Expecting value" in resp.json["message"] def test_get_entity_by_id(client, pg_driver, cgci_blgsp, submitter): post_example_entities_together(client, submitter) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = '/v0/submission/CGCI/BLGSP/entities/{case_id}'.format(case_id=case_id) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/entities/{case_id}".format(case_id=case_id) + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.json['entities'][0]['properties']['id'] == case_id, r.data + assert r.json["entities"][0]["properties"]["id"] == case_id, r.data def test_invalid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter): @@ -565,31 +573,37 @@ def test_invalid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitte Test that submitting an invalid data file doesn't create an index and an alias. """ + def fail_index_test(_): - raise AssertionError('IndexClient tried to create index or alias') + raise AssertionError("IndexClient tried to create index or alias") # Since the IndexClient should never be called to register anything if the # file is invalid, change the ``create`` and ``create_alias`` methods to # raise an error. monkeypatch.setattr( - UploadTransaction, 'signpost.create', fail_index_test, raising=False + UploadTransaction, "signpost.create", fail_index_test, raising=False ) monkeypatch.setattr( - UploadTransaction, 'signpost.create_alias', fail_index_test, - raising=False + UploadTransaction, "signpost.create_alias", fail_index_test, raising=False ) # Attempt to post the invalid entities. - test_fnames = ( - data_fnames - + ['read_group.json', 'submitted_unaligned_reads_invalid.json'] - ) - resp = post_example_entities_together( - client, submitter, data_fnames2=test_fnames - ) + test_fnames = data_fnames + [ + "read_group.json", + "submitted_unaligned_reads_invalid.json", + ] + resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) print(resp) -def test_valid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter, index_client, require_index_exists_off): +def test_valid_file_index( + monkeypatch, + client, + pg_driver, + cgci_blgsp, + submitter, + index_client, + require_index_exists_off, +): """ Test that submitting a valid data file creates an index and an alias. """ @@ -598,54 +612,45 @@ def test_valid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter, # called. # Attempt to post the valid entities. - test_fnames = ( - data_fnames - + ['read_group.json', 'submitted_unaligned_reads.json'] - ) - resp = post_example_entities_together( - client, submitter, data_fnames2=test_fnames - ) + test_fnames = data_fnames + ["read_group.json", "submitted_unaligned_reads.json"] + resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) assert resp.status_code == 201, resp.data # this is a node that will have an indexd entry sur_entity = None - for entity in resp.json['entities']: - if entity['type'] == 'submitted_unaligned_reads': + for entity in resp.json["entities"]: + if entity["type"] == "submitted_unaligned_reads": sur_entity = entity - path = '/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}'.format(nid=sur_entity['id']) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format( + nid=sur_entity["id"] + ) + r = client.get(path, headers=submitter) - print r.json + print(r.json) data = r.json - object_id = data[0]['object_id'] + object_id = data[0]["object_id"] assert len(data) == 1 assert object_id - assert sur_entity, 'No submitted_unaligned_reads entity created' - assert index_client.get(object_id), 'No indexd document created' + assert sur_entity, "No submitted_unaligned_reads entity created" + assert index_client.get(object_id), "No indexd document created" def test_export_entity_by_id(client, pg_driver, cgci_blgsp, submitter): post_example_entities_together(client, submitter) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = '/v0/submission/CGCI/BLGSP/export/?ids={case_id}'.format(case_id=case_id) - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?ids={case_id}".format(case_id=case_id) + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') + assert r.headers["Content-Disposition"].endswith("tsv") - path += '&format=json' - r = client.get( - path, - headers=submitter) + path += "&format=json" + r = client.get(path, headers=submitter) data = r.json assert len(data) == 1 - assert data[0]['id'] == case_id + assert data[0]["id"] == case_id def test_export_all_node_types(client, pg_driver, cgci_blgsp, submitter): @@ -654,19 +659,17 @@ def test_export_all_node_types(client, pg_driver, cgci_blgsp, submitter): case = pg_driver.nodes(md.Case).first() new_case = md.Case(str(uuid.uuid4())) new_case.props = case.props - new_case.submitter_id = 'case-2' + new_case.submitter_id = "case-2" s.add(new_case) case_count = pg_driver.nodes(md.Case).count() - path = '/v0/submission/CGCI/BLGSP/export/?node_label=case' - r = client.get( - path, - headers=submitter) + path = "/v0/submission/CGCI/BLGSP/export/?node_label=case" + r = client.get(path, headers=submitter) assert r.status_code == 200, r.data - assert r.headers['Content-Disposition'].endswith('tsv') - assert len(r.data.strip().split('\n')) == case_count + 1 + assert r.headers["Content-Disposition"].endswith("tsv") + assert len(r.data.strip().split("\n")) == case_count + 1 -@pytest.mark.parametrize('file_type', ['json', 'tsv']) +@pytest.mark.parametrize("file_type", ["json", "tsv"]) def test_export_error(client, cgci_blgsp, submitter, file_type): """ Certain node types and categories cant't be exported through the @@ -674,12 +677,11 @@ def test_export_error(client, cgci_blgsp, submitter, file_type): Test that trying to export using these as the node_label argument fails with 400. """ - for node_label in ['root', '_all']: + for node_label in ["root", "_all"]: path = ( - '/v0/submission/CGCI/BLGSP/export/' - '?file_format={}' - '&node_label={}' - .format(file_type, node_label) + "/v0/submission/CGCI/BLGSP/export/" + "?file_format={}" + "&node_label={}".format(file_type, node_label) ) response = client.get(path, headers=submitter) assert response.status_code == 400 @@ -691,14 +693,18 @@ def test_delete_non_empty_project(client, pg_driver, cgci_blgsp, submitter, admi """ headers = submitter resp = client.put( - BLGSP_PATH, headers=headers, data=json.dumps({ - "type": "experiment", - "submitter_id": "BLGSP-71-06-00019", - "projects": { - "id": "daa208a7-f57a-562c-a04a-7a7c77542c98" - }})) + BLGSP_PATH, + headers=headers, + data=json.dumps( + { + "type": "experiment", + "submitter_id": "BLGSP-71-06-00019", + "projects": {"id": "daa208a7-f57a-562c-a04a-7a7c77542c98"}, + } + ), + ) - path = '/v0/submission/CGCI/BLGSP' + path = "/v0/submission/CGCI/BLGSP" resp = client.delete(path, headers=admin) assert resp.status_code == 400 @@ -707,7 +713,7 @@ def test_delete_project_without_admin_token(client, pg_driver, cgci_blgsp, membe """ Test that returns error when attemping to delete non-empty project """ - path = '/v0/submission/CGCI/BLGSP' + path = "/v0/submission/CGCI/BLGSP" resp = client.delete(path, headers=member) assert resp.status_code == 403 @@ -716,7 +722,7 @@ def test_delete_non_existed_project(client, pg_driver, cgci_blgsp, submitter, ad """ Test that returns error when attemping to delete a non-existed project """ - path = '/v0/submission/CGCI/NOT_EXIST' + path = "/v0/submission/CGCI/NOT_EXIST" resp = client.delete(path, headers=admin) assert resp.status_code == 404 @@ -725,14 +731,11 @@ def test_delete_empty_project(client, pg_driver, cgci_blgsp, submitter, admin): """ Test that successfully deletes an empty project """ - path = '/v0/submission/CGCI/BLGSP' + path = "/v0/submission/CGCI/BLGSP" resp = client.delete(path, headers=admin) assert resp.status_code == 204 with flask.current_app.db.session_scope(): - project = ( - flask.current_app. - db.nodes(md.Project). - props(code='BLGSP').first()) + project = flask.current_app.db.nodes(md.Project).props(code="BLGSP").first() assert not project @@ -740,7 +743,7 @@ def test_delete_empty_non_program(client, pg_driver, cgci_blgsp, admin): """ Test that return error when attempting to delete a non-empty program """ - path = '/v0/submission/CGCI' + path = "/v0/submission/CGCI" resp = client.delete(path, headers=admin) assert resp.status_code == 400 @@ -750,7 +753,7 @@ def test_delete_program_without_admin_token(client, pg_driver, admin, member): Test that returns error since the client does not have privillege to delele the program """ - path = '/v0/submission/CGCI' + path = "/v0/submission/CGCI" put_cgci(client, admin) resp = client.delete(path, headers=member) assert resp.status_code == 403 @@ -760,15 +763,12 @@ def test_delete_program(client, pg_driver, admin): """ Test that successfully deletes an empty program """ - path = '/v0/submission/CGCI' + path = "/v0/submission/CGCI" put_cgci(client, admin) resp = client.delete(path, headers=admin) assert resp.status_code == 204 with flask.current_app.db.session_scope(): - program = ( - flask.current_app. - db.nodes(md.Program). - props(name='CGCI').first()) + program = flask.current_app.db.nodes(md.Program).props(name="CGCI").first() assert not program @@ -778,11 +778,10 @@ def test_update_program_without_admin_token(client, pg_driver, admin, member): privilege to update the program """ put_cgci(client, admin) - data = json.dumps({ - 'name': 'CGCI', 'type': 'program', - 'dbgap_accession_number': 'phs000235_2' - }) - resp = client.put('/v0/submission', headers=member, data=data) + data = json.dumps( + {"name": "CGCI", "type": "program", "dbgap_accession_number": "phs000235_2"} + ) + resp = client.put("/v0/submission", headers=member, data=data) assert resp.status_code == 403 @@ -791,15 +790,11 @@ def test_update_program(client, pg_driver, admin): Test that successfully updates a program """ put_cgci(client, admin) - data = json.dumps({ - 'name': 'CGCI', 'type': 'program', - 'dbgap_accession_number': 'phs000235_2' - }) - resp = client.put('/v0/submission', headers=admin, data=data) + data = json.dumps( + {"name": "CGCI", "type": "program", "dbgap_accession_number": "phs000235_2"} + ) + resp = client.put("/v0/submission", headers=admin, data=data) assert resp.status_code == 200 with flask.current_app.db.session_scope(): - program = ( - flask.current_app. - db.nodes(md.Program). - props(name='CGCI').first()) - assert program.props['dbgap_accession_number'] == 'phs000235_2' + program = flask.current_app.db.nodes(md.Program).props(name="CGCI").first() + assert program.props["dbgap_accession_number"] == "phs000235_2" From b85496284a1c6964cebed1121e9f3b2f6fc5b17f Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Mon, 11 Feb 2019 21:17:14 -0600 Subject: [PATCH 5/7] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index edd0b923..833a6479 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ xmltodict==0.9.2 -e git+https://git@github.com/uc-cdis/cdis_oauth2client.git@0.1.3#egg=cdis_oauth2client -e git+https://git@github.com/uc-cdis/cdis-python-utils.git@0.2.8#egg=cdispyutils -e git+https://git@github.com/uc-cdis/datamodelutils.git@0.4.0#egg=datamodelutils --e git+https://git@github.com/uc-cdis/psqlgraph.git@3b5ac0402857f990e93d8c32ced5264592360d27#egg=psqlgraph +-e git+https://git@github.com/uc-cdis/psqlgraph.git@1.2.2#egg=psqlgraph -e git+https://git@github.com/NCI-GDC/signpost.git@v1.1#egg=signpost # required for gdcdatamodel, not required for sheepdog -e git+https://git@github.com/uc-cdis/cdiserrors.git@0.1.1#egg=cdiserrors From 50e85cb4e8b119376b086e773e1448ea68402ef0 Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Mon, 11 Feb 2019 21:18:54 -0600 Subject: [PATCH 6/7] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 833a6479..aa816cac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ xmltodict==0.9.2 -e git+https://git@github.com/uc-cdis/cdislogging.git@master#egg=cdislogging -e git+https://git@github.com/NCI-GDC/cdisutils.git@f54e393c89939b2200dfae45c6235cbe2bae1206#egg=cdisutils -e git+https://git@github.com/uc-cdis/datadictionary.git@0.2.1#egg=gdcdictionary --e git+https://git@github.com/uc-cdis/gdcdatamodel.git@26c3da68c29edcba027fa7ca62a4c75c1886bacc#egg=gdcdatamodel +-e git+https://git@github.com/uc-cdis/gdcdatamodel.git@1.3.8#egg=gdcdatamodel -e git+https://git@github.com/uc-cdis/indexclient.git@1.5.7#egg=indexclient -e git+https://git@github.com/NCI-GDC/python-signpostclient.git@ca686f55772e9a7f839b4506090e7d2bb0de5f15#egg=signpostclient -e git+https://git@github.com/uc-cdis/storage-client.git@0.1.7#egg=storageclient From 6a92eb4c9740a1ca6b0bf5cd3d26035767254652 Mon Sep 17 00:00:00 2001 From: Zac Flamig Date: Mon, 11 Feb 2019 21:28:30 -0600 Subject: [PATCH 7/7] Update dev-requirements.txt --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 19016bf5..f0afca83 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,7 @@ mock==1.0.1 pytest-flask==0.11.0 moto==0.4.5 sphinxcontrib-httpdomain==1.3.0 +more-itertools>=4.0.0,<6.0.0 # force install of sqlalchemy 0.9.9 because indexd installs 1.0.8 # this new version is not backwards compatible with 0.9.9