Update requirements.txt

bin/setup_psqlgraph.py

@@ -5,13 +5,13 @@
 from psqlgraph import create_all, Node, Edge
 
 
-def try_drop_test_data(user, database, root_user='postgres', host='', root_password='' ):
-    print('Dropping old test data')
-    connect_str="postgres://{user}@{host}/postgres".format(
-        user=root_user, host=host)
+def try_drop_test_data(user, database, root_user="postgres", host="", root_password=""):
+    print("Dropping old test data")
+    connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host)
     if root_password:
-      connect_str="postgres://{user}:{password}@{host}/postgres".format(
-          user=root_user, password=root_password, host=host)
+        connect_str = "postgres://{user}:{password}@{host}/postgres".format(
+            user=root_user, password=root_password, host=host
+        )
 
     engine = create_engine(connect_str)
 
@@ -21,28 +21,35 @@ def try_drop_test_data(user, database, root_user='postgres', host='', root_passw
     try:
         create_stmt = 'DROP DATABASE "{database}"'.format(database=database)
         conn.execute(create_stmt)
-    except Exception, msg:
+    except Exception as msg:
         logging.warning("Unable to drop test data:" + str(msg))
 
     conn.close()
 
 
-def setup_database(user, password, database, root_user='postgres',
-                   host='', no_drop=False, no_user=False, root_password=''
-                  ):
+def setup_database(
+    user,
+    password,
+    database,
+    root_user="postgres",
+    host="",
+    no_drop=False,
+    no_user=False,
+    root_password="",
+):
     """
     setup the user and database
     """
-    print('Setting up test database')
+    print("Setting up test database")
 
     if not no_drop:
         try_drop_test_data(user, database, root_user, host, root_password)
 
-    connect_str="postgres://{user}@{host}/postgres".format(
-        user=root_user, host=host)
+    connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host)
     if password:
-      connect_str="postgres://{user}:{password}@{host}/postgres".format(
-          user=root_user, password=root_password, host=host)
+        connect_str = "postgres://{user}:{password}@{host}/postgres".format(
+            user=root_user, password=root_password, host=host
+        )
 
     engine = create_engine(connect_str)
     conn = engine.connect()
@@ -51,23 +58,26 @@ def setup_database(user, password, database, root_user='postgres',
     create_stmt = 'CREATE DATABASE "{database}"'.format(database=database)
     try:
         conn.execute(create_stmt)
-    except Exception, msg:
-        logging.warn('Unable to create database: {}'.format(msg))
+    except Exception as msg:
+        logging.warn("Unable to create database: {}".format(msg))
 
     if not no_user:
         try:
             user_stmt = "CREATE USER {user} WITH PASSWORD '{password}'".format(
-                user=user, password=password)
+                user=user, password=password
+            )
             conn.execute(user_stmt)
-        except Exception, msg:
+        except Exception as msg:
             logging.warn("Unable to add user:" + str(msg))
         # User may already exist - GRANT privs on new db
         try:
-            perm_stmt = 'GRANT ALL PRIVILEGES ON DATABASE {database} to {password}'\
-                        ''.format(database=database, password=password)
+            perm_stmt = (
+                "GRANT ALL PRIVILEGES ON DATABASE {database} to {password}"
+                "".format(database=database, password=password)
+            )
             conn.execute(perm_stmt)
             conn.execute("commit")
-        except Exception, msg:
+        except Exception as msg:
             logging.warn("Unable to GRANT privs to user:" + str(msg))
     conn.close()
 
@@ -76,57 +86,80 @@ def create_tables(host, user, password, database):
     """
     create a table
     """
-    print('Creating tables in test database')
+    print("Creating tables in test database")
 
-    engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
-        user=user, host=host, pwd=password, db=database))
+    engine = create_engine(
+        "postgres://{user}:{pwd}@{host}/{db}".format(
+            user=user, host=host, pwd=password, db=database
+        )
+    )
     create_all(engine)
     versioned_nodes.Base.metadata.create_all(engine)
 
 
 def create_indexes(host, user, password, database):
-    print('Creating indexes')
-    engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
-        user=user, host=host, pwd=password, db=database))
+    print("Creating indexes")
+    engine = create_engine(
+        "postgres://{user}:{pwd}@{host}/{db}".format(
+            user=user, host=host, pwd=password, db=database
+        )
+    )
     index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c]
     for scls in Node.get_subclasses():
         tablename = scls.__tablename__
-        map(engine.execute, index(
-            tablename, [
-                'node_id',
-            ]))
-        map(engine.execute, [
-            "CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
-            "CREATE INDEX ON {} USING gin (_props)".format(tablename),
-            "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename),
-        ])
+        map(engine.execute, index(tablename, ["node_id"]))
+        map(
+            engine.execute,
+            [
+                "CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
+                "CREATE INDEX ON {} USING gin (_props)".format(tablename),
+                "CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename),
+            ],
+        )
     for scls in Edge.get_subclasses():
-        map(engine.execute, index(
-            scls.__tablename__, [
-                'src_id',
-                'dst_id',
-                'dst_id, src_id',
-            ]))
+        map(
+            engine.execute,
+            index(scls.__tablename__, ["src_id", "dst_id", "dst_id, src_id"]),
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--host", type=str, action="store",
-                        default='localhost', help="psql-server host")
-    parser.add_argument("--user", type=str, action="store",
-                        default='test', help="psql test user")
-    parser.add_argument("--password", type=str, action="store",
-                        default='test', help="psql test password")
-    parser.add_argument("--database", type=str, action="store",
-                        default='sheepdog_automated_test', help="psql test database")
-    parser.add_argument("--no-drop", action="store_true",
-                        default=False, help="do not drop any data")
-    parser.add_argument("--no-user", action="store_true",
-                        default=False, help="do not create user")
+    parser.add_argument(
+        "--host", type=str, action="store", default="localhost", help="psql-server host"
+    )
+    parser.add_argument(
+        "--user", type=str, action="store", default="test", help="psql test user"
+    )
+    parser.add_argument(
+        "--password",
+        type=str,
+        action="store",
+        default="test",
+        help="psql test password",
+    )
+    parser.add_argument(
+        "--database",
+        type=str,
+        action="store",
+        default="sheepdog_automated_test",
+        help="psql test database",
+    )
+    parser.add_argument(
+        "--no-drop", action="store_true", default=False, help="do not drop any data"
+    )
+    parser.add_argument(
+        "--no-user", action="store_true", default=False, help="do not create user"
+    )
 
     args = parser.parse_args()
-    setup_database(args.user, args.password, args.database,
-                   no_drop=args.no_drop, no_user=args.no_user)
+    setup_database(
+        args.user,
+        args.password,
+        args.database,
+        no_drop=args.no_drop,
+        no_user=args.no_user,
+    )
     create_tables(args.host, args.user, args.password, args.database)
     create_indexes(args.host, args.user, args.password, args.database)

dev-requirements.txt

@@ -8,6 +8,7 @@ mock==1.0.1
 pytest-flask==0.11.0
 moto==0.4.5
 sphinxcontrib-httpdomain==1.3.0
+more-itertools>=4.0.0,<6.0.0
 
 # force install of sqlalchemy 0.9.9 because indexd installs 1.0.8
 # this new version is not backwards compatible with 0.9.9

requirements.txt

@@ -24,14 +24,14 @@ more-itertools==5.0.0
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=cdis_oauth2client
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=cdispyutils
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=datamodelutils
--e git+https://[email protected]/NCI-GDC/[email protected].0#egg=psqlgraph
+-e git+https://[email protected]/uc-cdis/[email protected].2#egg=psqlgraph
 -e git+https://[email protected]/NCI-GDC/[email protected]#egg=signpost
 # required for gdcdatamodel, not required for sheepdog
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=cdiserrors
 -e git+https://[email protected]/uc-cdis/cdislogging.git@master#egg=cdislogging
 -e git+https://[email protected]/NCI-GDC/cdisutils.git@f54e393c89939b2200dfae45c6235cbe2bae1206#egg=cdisutils
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=gdcdictionary
--e git+https://[email protected]/uc-cdis/[email protected].7#egg=gdcdatamodel
+-e git+https://[email protected]/uc-cdis/[email protected].8#egg=gdcdatamodel
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=indexclient
 -e git+https://[email protected]/NCI-GDC/python-signpostclient.git@ca686f55772e9a7f839b4506090e7d2bb0de5f15#egg=signpostclient
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=storageclient

sheepdog/api.py

@@ -128,6 +128,9 @@ def app_init(app):
     app.config["USE_DBGAP"] = False
     app.config["IS_GDC"] = False
 
+    # Turn off for performance
+    app.config["JSONIFY_PRETTYPRINT_REGULAR"] = False
+
     # default settings
     app.config["AUTO_MIGRATE_DATABASE"] = app.config.get("AUTO_MIGRATE_DATABASE", True)
     app.config["REQUIRE_FILE_INDEX_EXISTS"] = (