diff --git a/noxfile.py b/noxfile.py index 625b0e39..26b5540b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,11 +18,12 @@ from __future__ import absolute_import +from functools import wraps import os import pathlib import re -import re import shutil +import time from typing import Dict, List import warnings @@ -48,6 +49,7 @@ "asyncmock", "pytest", "pytest-cov", + "pytest-xdist", "pytest-asyncio", ] UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] @@ -79,6 +81,7 @@ "mock", "pytest", "google-cloud-testutils", + "pytest-xdist", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] @@ -106,6 +109,26 @@ CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +def _calculate_duration(func): + """This decorator prints the execution time for the decorated function.""" + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.monotonic() + result = func(*args, **kwargs) + end = time.monotonic() + total_seconds = round(end - start) + hours = total_seconds // 3600 # Integer division to get hours + remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds + minutes = remaining_seconds // 60 + seconds = remaining_seconds % 60 + human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}" + print(f"Session ran in {total_seconds} seconds ({human_time})") + return result + + return wrapper + + nox.options.sessions = [ "unit", "system", @@ -124,6 +147,7 @@ @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint(session): """Run linters. @@ -140,6 +164,7 @@ def lint(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) @@ -150,6 +175,7 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def format(session): """ Run isort to sort imports. Then run black @@ -170,6 +196,7 @@ def format(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") @@ -205,6 +232,7 @@ def install_unittest_dependencies(session, *constraints): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@_calculate_duration @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], @@ -237,6 +265,7 @@ def unit(session, protobuf_implementation, install_extras=True): # Run py.test against the unit tests. session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=sqlalchemy_bigquery", @@ -284,6 +313,7 @@ def install_systemtest_dependencies(session, *constraints): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system(session): """Run the system test suite.""" constraints_path = str( @@ -311,6 +341,7 @@ def system(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -319,6 +350,7 @@ def system(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -327,6 +359,7 @@ def system(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system_noextras(session): """Run the system test suite.""" constraints_path = str( @@ -356,6 +389,7 @@ def system_noextras(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -364,6 +398,7 @@ def system_noextras(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -372,6 +407,7 @@ def system_noextras(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@_calculate_duration def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( @@ -389,6 +425,7 @@ def compliance(session): session.install( "mock", "pytest", + "pytest-xdist", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -406,10 +443,11 @@ def compliance(session): session.run( "py.test", + "-n=2", "-vv", f"--junitxml=compliance_{session.python}_sponge_log.xml", - "--reruns=3", - "--reruns-delay=60", + "--reruns=0", + "--reruns-delay=1", "--only-rerun=Exceeded rate limits", "--only-rerun=Already Exists", "--only-rerun=Not found", @@ -426,6 +464,7 @@ def compliance(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def cover(session): """Run the final coverage report. @@ -439,6 +478,7 @@ def cover(session): @nox.session(python="3.10") +@_calculate_duration def docs(session): """Build the docs for this library.""" @@ -476,6 +516,7 @@ def docs(session): @nox.session(python="3.10") +@_calculate_duration def docfx(session): """Build the docfx yaml files for this library.""" @@ -524,6 +565,7 @@ def docfx(session): @nox.session(python="3.12") +@_calculate_duration @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], @@ -612,6 +654,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -623,6 +666,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_folder_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 0204bc92..61c41b2d 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -69,6 +69,25 @@ TABLE_VALUED_ALIAS_ALIASES = "bigquery_table_valued_alias_aliases" +from sqlalchemy.testing.provision import create_db, drop_db +@create_db.for_db("bigquery") +def _bigquery_create_db(cfg, eng, ident): + # This and _bigquery_drop_db are adequate to get past the errors generated when using xdist: + # NotImplementedError: no DB creation routine for cfg + # NotImplementedError: no DB drop routine for cfg + # + # HOWEVER... we need to populate this in some way to create dbs that + # work. THere are examples of create_db style functions in various + # dialects provision.py files. + # This sqlalchemy issue speaks to this problem: + # https://github.com/sqlalchemy/sqlalchemy/discussions/10948 + pass + +@drop_db.for_db("bigquery") +def _bigquery_drop_db(cfg, eng, ident): + pass # _drop_dbs_w_ident(eng.url.database, eng.driver, ident) + # SEE note above for create_db + def assert_(cond, message="Assertion failed"): # pragma: NO COVER if not cond: