From 169265c57d6c44975e07efdb6f9939d747528c28 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 4 Feb 2025 17:20:29 -0500 Subject: [PATCH 1/4] standardize pytest config and env files --- dbt-adapters/pyproject.toml | 11 +++-------- dbt-athena-community/pyproject.toml | 9 ++++----- dbt-athena/pyproject.toml | 5 ++--- dbt-bigquery/{test.env.example => .env.example} | 0 dbt-bigquery/pyproject.toml | 6 ++---- dbt-postgres/pyproject.toml | 8 +++----- dbt-redshift/{test.env.example => .env.example} | 0 dbt-redshift/pyproject.toml | 9 ++++----- dbt-snowflake/{test.env.example => .env.example} | 0 dbt-snowflake/pyproject.toml | 3 +-- dbt-spark/{test.env.example => .env.example} | 0 dbt-spark/pyproject.toml | 3 +-- 12 files changed, 20 insertions(+), 34 deletions(-) rename dbt-bigquery/{test.env.example => .env.example} (100%) rename dbt-redshift/{test.env.example => .env.example} (100%) rename dbt-snowflake/{test.env.example => .env.example} (100%) rename dbt-spark/{test.env.example => .env.example} (100%) diff --git a/dbt-adapters/pyproject.toml b/dbt-adapters/pyproject.toml index 50941be2d..67ed88070 100644 --- a/dbt-adapters/pyproject.toml +++ b/dbt-adapters/pyproject.toml @@ -48,11 +48,6 @@ mypy_path = "third-party-stubs/" module = ["dbt.adapters.events.adapter_types_pb2"] follow_imports = "skip" -[tool.pytest] -env_files = ["test.env"] -testpaths = [ - "tests/functional", - "tests/unit", -] -color = true -csv = "results.csv" +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/functional"] +addopts = "-v --color=yes -n auto" diff --git a/dbt-athena-community/pyproject.toml b/dbt-athena-community/pyproject.toml index b5de59bbf..0c8367224 100644 --- a/dbt-athena-community/pyproject.toml +++ b/dbt-athena-community/pyproject.toml @@ -34,11 +34,10 @@ Documentation = "https://docs.getdbt.com" Repository = "https://github.com/dbt-labs/dbt-athena.git#subdirectory=dbt-athena" Issues = "https://github.com/dbt-labs/dbt-athena/issues" -[tool.pytest] -testpaths = [ - "../dbt-athena/tests/unit", - "../dbt-athena/tests/functional", -] +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/functional"] +env_files = ["../dbt-athena/.env"] +addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 69ec0018b..80effa672 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -45,10 +45,9 @@ Repository = "https://github.com/dbt-labs/dbt-athena.git#subdirectory=dbt-athena Issues = "https://github.com/dbt-labs/dbt-athena/issues" Changelog = "https://github.com/dbt-labs/dbt-athena/blob/main/dbt-athena/CHANGELOG.md" -[tool.pytest] +[tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] -color = true -csv = "results.csv" +addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-bigquery/test.env.example b/dbt-bigquery/.env.example similarity index 100% rename from dbt-bigquery/test.env.example rename to dbt-bigquery/.env.example diff --git a/dbt-bigquery/pyproject.toml b/dbt-bigquery/pyproject.toml index b2d55b25f..5b72cf378 100644 --- a/dbt-bigquery/pyproject.toml +++ b/dbt-bigquery/pyproject.toml @@ -47,10 +47,8 @@ Changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/main/CHANGELOG.md" mypy_path = "third-party-stubs/" [tool.pytest.ini_options] -testpaths = ["tests/functional", "tests/unit"] -env_files = ["test.env"] -addopts = "-v -n auto" -color = true +testpaths = ["tests/unit", "tests/functional"] +addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-postgres/pyproject.toml b/dbt-postgres/pyproject.toml index 6f4b4604a..79d691fb6 100644 --- a/dbt-postgres/pyproject.toml +++ b/dbt-postgres/pyproject.toml @@ -42,8 +42,6 @@ Repository = "https://github.com/dbt-labs/dbt-postgres.git" Issues = "https://github.com/dbt-labs/dbt-postgres/issues" Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" -[tool.pytest] -testpaths = [ - "tests/functional", - "tests/unit", -] +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/functional"] +addopts = "-v --color=yes -n auto" diff --git a/dbt-redshift/test.env.example b/dbt-redshift/.env.example similarity index 100% rename from dbt-redshift/test.env.example rename to dbt-redshift/.env.example diff --git a/dbt-redshift/pyproject.toml b/dbt-redshift/pyproject.toml index 3f409d896..b318cdfac 100644 --- a/dbt-redshift/pyproject.toml +++ b/dbt-redshift/pyproject.toml @@ -48,13 +48,12 @@ Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" mypy_path = "third-party-stubs/" [tool.pytest.ini_options] -testpaths = ["tests/functional", "tests/unit"] -env_files = ["test.env"] +testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +markers = [ + 'flaky: marks tests as flaky so they run one at a time (de-select with `-m "not flaky"`)' +] filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] -markers = [ - 'flaky: marks tests as flaky so they run one at a time (de-select with `-m "not flaky"`)' -] diff --git a/dbt-snowflake/test.env.example b/dbt-snowflake/.env.example similarity index 100% rename from dbt-snowflake/test.env.example rename to dbt-snowflake/.env.example diff --git a/dbt-snowflake/pyproject.toml b/dbt-snowflake/pyproject.toml index 568aa3533..d477035bf 100644 --- a/dbt-snowflake/pyproject.toml +++ b/dbt-snowflake/pyproject.toml @@ -40,8 +40,7 @@ Issues = "https://github.com/dbt-labs/dbt-snowflake/issues" Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" [tool.pytest.ini_options] -testpaths = ["tests/functional", "tests/unit"] -env_files = ["test.env"] +testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", diff --git a/dbt-spark/test.env.example b/dbt-spark/.env.example similarity index 100% rename from dbt-spark/test.env.example rename to dbt-spark/.env.example diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index de27f9cb2..270d8dbf4 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -51,8 +51,7 @@ Issues = "https://github.com/dbt-labs/dbt-spark/issues" Changelog = "https://github.com/dbt-labs/dbt-spark/blob/main/CHANGELOG.md" [tool.pytest.ini_options] -testpaths = ["tests/functional", "tests/unit"] -env_files = ["test.env"] +testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", From 19fb3e3cc8f3fbb27b78ab948be55b74120ce916 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 4 Feb 2025 18:40:22 -0500 Subject: [PATCH 2/4] update .env to test.env --- dbt-athena-community/pyproject.toml | 2 +- dbt-athena/pyproject.toml | 1 + dbt-athena/{.env.example => test.env.example} | 0 dbt-bigquery/pyproject.toml | 1 + dbt-bigquery/{.env.example => test.env.example} | 0 dbt-postgres/pyproject.toml | 1 + dbt-postgres/{.env.example => test.env.example} | 0 dbt-redshift/pyproject.toml | 1 + dbt-redshift/{.env.example => test.env.example} | 0 dbt-snowflake/pyproject.toml | 1 + dbt-snowflake/{.env.example => test.env.example} | 0 dbt-spark/pyproject.toml | 1 + dbt-spark/{.env.example => test.env.example} | 0 13 files changed, 7 insertions(+), 1 deletion(-) rename dbt-athena/{.env.example => test.env.example} (100%) rename dbt-bigquery/{.env.example => test.env.example} (100%) rename dbt-postgres/{.env.example => test.env.example} (100%) rename dbt-redshift/{.env.example => test.env.example} (100%) rename dbt-snowflake/{.env.example => test.env.example} (100%) rename dbt-spark/{.env.example => test.env.example} (100%) diff --git a/dbt-athena-community/pyproject.toml b/dbt-athena-community/pyproject.toml index a8c021bc7..4ea6b8497 100644 --- a/dbt-athena-community/pyproject.toml +++ b/dbt-athena-community/pyproject.toml @@ -37,7 +37,7 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-athena/CHANG [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] -env_files = ["../dbt-athena/.env"] +env_files = ["../dbt-athena/test.env"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 3a26e9743..c92d43d18 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -48,6 +48,7 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-athena/CHANG [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-athena/.env.example b/dbt-athena/test.env.example similarity index 100% rename from dbt-athena/.env.example rename to dbt-athena/test.env.example diff --git a/dbt-bigquery/pyproject.toml b/dbt-bigquery/pyproject.toml index 73f9bc344..aa180b3a9 100644 --- a/dbt-bigquery/pyproject.toml +++ b/dbt-bigquery/pyproject.toml @@ -49,6 +49,7 @@ mypy_path = "third-party-stubs/" [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-bigquery/.env.example b/dbt-bigquery/test.env.example similarity index 100% rename from dbt-bigquery/.env.example rename to dbt-bigquery/test.env.example diff --git a/dbt-postgres/pyproject.toml b/dbt-postgres/pyproject.toml index 14138ae85..fcf8f8318 100644 --- a/dbt-postgres/pyproject.toml +++ b/dbt-postgres/pyproject.toml @@ -45,3 +45,4 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-postgres/CHA [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] diff --git a/dbt-postgres/.env.example b/dbt-postgres/test.env.example similarity index 100% rename from dbt-postgres/.env.example rename to dbt-postgres/test.env.example diff --git a/dbt-redshift/pyproject.toml b/dbt-redshift/pyproject.toml index 6a95ab22f..c6a799871 100644 --- a/dbt-redshift/pyproject.toml +++ b/dbt-redshift/pyproject.toml @@ -50,6 +50,7 @@ mypy_path = "third-party-stubs/" [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] markers = [ 'flaky: marks tests as flaky so they run one at a time (de-select with `-m "not flaky"`)' ] diff --git a/dbt-redshift/.env.example b/dbt-redshift/test.env.example similarity index 100% rename from dbt-redshift/.env.example rename to dbt-redshift/test.env.example diff --git a/dbt-snowflake/pyproject.toml b/dbt-snowflake/pyproject.toml index acf696668..9251373a1 100644 --- a/dbt-snowflake/pyproject.toml +++ b/dbt-snowflake/pyproject.toml @@ -42,6 +42,7 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-snowflake/CH [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] diff --git a/dbt-snowflake/.env.example b/dbt-snowflake/test.env.example similarity index 100% rename from dbt-snowflake/.env.example rename to dbt-snowflake/test.env.example diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index 1877f287b..9ff436422 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -53,6 +53,7 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-spark/CHANGE [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes -n auto" +env_files = ["test.env"] filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-spark/.env.example b/dbt-spark/test.env.example similarity index 100% rename from dbt-spark/.env.example rename to dbt-spark/test.env.example From 29eaebc0c1a669f41a72f6b034481bbde1368e3c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Feb 2025 15:25:12 -0500 Subject: [PATCH 3/4] update filters to be more specific; avoid testing on things like docs changes to minimize collisions in CI across pull requests --- .github/filters/changelog-entry-check.yml | 24 ++++--- .github/filters/integration-tests.yml | 73 ++++++++++++++------- .github/filters/unit-tests.yml | 77 ++++++++++++++++++----- .github/filters/verify-build.yml | 66 +++++++++++++------ 4 files changed, 173 insertions(+), 67 deletions(-) diff --git a/.github/filters/changelog-entry-check.yml b/.github/filters/changelog-entry-check.yml index 8ebf947ce..e47727200 100644 --- a/.github/filters/changelog-entry-check.yml +++ b/.github/filters/changelog-entry-check.yml @@ -1,16 +1,24 @@ dbt-adapters: -- 'dbt-adapters/**' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-tests-adapter: -- 'dbt-tests-adapter/**' +- 'dbt-tests-adapter/src/**' +- 'dbt-tests-adapter/pyproject.toml' dbt-athena: -- 'dbt-athena/**' +- 'dbt-athena/src/**' +- 'dbt-athena/pyproject.toml' dbt-bigquery: -- 'dbt-bigquery/**' +- 'dbt-bigquery/src/**' +- 'dbt-bigquery/pyproject.toml' dbt-postgres: -- 'dbt-postgres/**' +- 'dbt-postgres/src/**' +- 'dbt-postgres/pyproject.toml' dbt-redshift: -- 'dbt-redshift/**' +- 'dbt-redshift/src/**' +- 'dbt-redshift/pyproject.toml' dbt-snowflake: -- 'dbt-snowflake/**' +- 'dbt-snowflake/src/**' +- 'dbt-snowflake/pyproject.toml' dbt-spark: -- 'dbt-spark/**' +- 'dbt-spark/src/**' +- 'dbt-spark/pyproject.toml' diff --git a/.github/filters/integration-tests.yml b/.github/filters/integration-tests.yml index a985abf8b..8a052cdfe 100644 --- a/.github/filters/integration-tests.yml +++ b/.github/filters/integration-tests.yml @@ -1,30 +1,57 @@ dbt-athena: -- 'dbt-athena/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-athena/src/**' +- 'dbt-athena/hatch.toml' +- 'dbt-athena/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-athena/tests/functional/**' +- 'dbt-athena/tests/__init__.py' dbt-athena-community: -- 'dbt-athena/**' -- 'dbt-athena-community/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-athena-community/src/**' +- 'dbt-athena-community/hatch.toml' +- 'dbt-athena-community/pyproject.toml' dbt-bigquery: -- 'dbt-bigquery/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-bigquery/src/**' +- 'dbt-bigquery/hatch.toml' +- 'dbt-bigquery/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-bigquery/tests/functional/**' +- 'dbt-bigquery/tests/__init__.py' +- 'dbt-bigquery/tests/conftest.py' dbt-postgres: -- 'dbt-postgres/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-postgres/src/**' +- 'dbt-postgres/hatch.toml' +- 'dbt-postgres/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-postgres/tests/functional/**' +- 'dbt-postgres/tests/conftest.py' dbt-redshift: -- 'dbt-redshift/**' -- 'dbt-postgres/src/dbt/include/postgres/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-redshift/src/**' +- 'dbt-redshift/hatch.toml' +- 'dbt-redshift/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-redshift/tests/functional/**' +- 'dbt-redshift/tests/__init__.py' +- 'dbt-redshift/tests/conftest.py' dbt-snowflake: -- 'dbt-snowflake/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-snowflake/src/**' +- 'dbt-snowflake/hatch.toml' +- 'dbt-snowflake/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-snowflake/tests/functional/**' +- 'dbt-snowflake/tests/__init__.py' +- 'dbt-snowflake/tests/conftest.py' dbt-spark: -- 'dbt-spark/**' -- 'dbt-adapters/**' -- 'dbt-tests-adapter/**' +- 'dbt-spark/src/**' +- 'dbt-spark/hatch.toml' +- 'dbt-spark/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-spark/tests/functional/**' +- 'dbt-spark/tests/__init__.py' +- 'dbt-spark/tests/conftest.py' +- 'dbt-spark/dagger/**' diff --git a/.github/filters/unit-tests.yml b/.github/filters/unit-tests.yml index 1399b1d75..dcfcca431 100644 --- a/.github/filters/unit-tests.yml +++ b/.github/filters/unit-tests.yml @@ -1,25 +1,68 @@ dbt-adapters: -- 'dbt-adapters/**' +- 'dbt-adapters/src/**' +- 'dbt-adapters/hatch.toml' +- 'dbt-adapters/pyproject.toml' +- 'dbt-adapters/tests/unit/**' +- 'dbt-adapters/tests/__init__.py' dbt-athena: -- 'dbt-athena/**' -- 'dbt-adapters/**' +- 'dbt-athena/src/**' +- 'dbt-athena/hatch.toml' +- 'dbt-athena/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-athena/tests/unit/**' +- 'dbt-athena/tests/__init__.py' dbt-athena-community: -- 'dbt-athena/**' -- 'dbt-athena-community/**' -- 'dbt-adapters/**' +- 'dbt-athena-community/src/**' +- 'dbt-athena-community/hatch.toml' +- 'dbt-athena-community/pyproject.toml' +- 'dbt-athena/src/**' +- 'dbt-athena/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-athena/tests/unit/**' +- 'dbt-athena/tests/__init__.py' dbt-bigquery: -- 'dbt-bigquery/**' -- 'dbt-adapters/**' +- 'dbt-bigquery/src/**' +- 'dbt-bigquery/hatch.toml' +- 'dbt-bigquery/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-bigquery/tests/unit/**' +- 'dbt-bigquery/tests/__init__.py' +- 'dbt-bigquery/tests/conftest.py' dbt-postgres: -- 'dbt-postgres/**' -- 'dbt-adapters/**' +- 'dbt-postgres/src/**' +- 'dbt-postgres/hatch.toml' +- 'dbt-postgres/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-postgres/tests/unit/**' +- 'dbt-postgres/tests/conftest.py' dbt-redshift: -- 'dbt-redshift/**' -- 'dbt-postgres/src/dbt/include/postgres/**' -- 'dbt-adapters/**' +- 'dbt-redshift/src/**' +- 'dbt-redshift/hatch.toml' +- 'dbt-redshift/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-redshift/tests/unit/**' +- 'dbt-redshift/tests/__init__.py' +- 'dbt-redshift/tests/conftest.py' dbt-snowflake: -- 'dbt-snowflake/**' -- 'dbt-adapters/**' +- 'dbt-snowflake/src/**' +- 'dbt-snowflake/hatch.toml' +- 'dbt-snowflake/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-snowflake/tests/unit/**' +- 'dbt-snowflake/tests/__init__.py' +- 'dbt-snowflake/tests/conftest.py' dbt-spark: -- 'dbt-spark/**' -- 'dbt-adapters/**' +- 'dbt-spark/src/**' +- 'dbt-spark/hatch.toml' +- 'dbt-spark/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' +- 'dbt-spark/tests/unit/**' +- 'dbt-spark/tests/__init__.py' +- 'dbt-spark/tests/conftest.py' diff --git a/.github/filters/verify-build.yml b/.github/filters/verify-build.yml index 1d8292dac..10acea790 100644 --- a/.github/filters/verify-build.yml +++ b/.github/filters/verify-build.yml @@ -1,28 +1,56 @@ dbt-adapters: -- 'dbt-adapters/**' +- 'dbt-adapters/src/**' +- 'dbt-adapters/hatch.toml' +- 'dbt-adapters/pyproject.toml' dbt-tests-adapter: -- 'dbt-tests-adapter/**' -- 'dbt-adapters/**' +- 'dbt-tests-adapter/src/**' +- 'dbt-tests-adapter/hatch.toml' +- 'dbt-tests-adapter/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-athena: -- 'dbt-athena/**' -- 'dbt-adapters/**' +- 'dbt-athena/src/**' +- 'dbt-athena/hatch.toml' +- 'dbt-athena/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-athena-community: -- 'dbt-athena/**' -- 'dbt-athena-community/**' -- 'dbt-adapters/**' +- 'dbt-athena-community/src/**' +- 'dbt-athena-community/hatch.toml' +- 'dbt-athena-community/pyproject.toml' +- 'dbt-athena/src/**' +- 'dbt-athena/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-bigquery: -- 'dbt-bigquery/**' -- 'dbt-adapters/**' +- 'dbt-bigquery/src/**' +- 'dbt-adapters/hatch.toml' +- 'dbt-bigquery/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-postgres: -- 'dbt-postgres/**' -- 'dbt-adapters/**' +- 'dbt-postgres/src/**' +- 'dbt-postgres/hatch.toml' +- 'dbt-postgres/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-redshift: -- 'dbt-redshift/**' -- 'dbt-postgres/**' -- 'dbt-adapters/**' +- 'dbt-redshift/src/**' +- 'dbt-redshift/hatch.toml' +- 'dbt-redshift/pyproject.toml' +- 'dbt-postgres/src/**' +- 'dbt-postgres/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-snowflake: -- 'dbt-snowflake/**' -- 'dbt-adapters/**' +- 'dbt-snowflake/src/**' +- 'dbt-snowflake/hatch.toml' +- 'dbt-snowflake/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' dbt-spark: -- 'dbt-spark/**' -- 'dbt-adapters/**' +- 'dbt-spark/src/**' +- 'dbt-spark/hatch.toml' +- 'dbt-spark/pyproject.toml' +- 'dbt-adapters/src/**' +- 'dbt-adapters/pyproject.toml' From ce2b10a671a0dd736554ea27ce4c37fb7ffde680 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Feb 2025 16:53:39 -0500 Subject: [PATCH 4/4] run integration tests in parallel in _integration-tests.yml --- .github/filters/integration-tests.yml | 4 -- .github/workflows/_integration-tests.yml | 86 +++++++++++++---------- .github/workflows/publish.yml | 2 +- .github/workflows/pull-request-checks.yml | 14 ++-- .github/workflows/scheduled-tests.yml | 17 +---- 5 files changed, 57 insertions(+), 66 deletions(-) diff --git a/.github/filters/integration-tests.yml b/.github/filters/integration-tests.yml index 8a052cdfe..8a2cfc6bb 100644 --- a/.github/filters/integration-tests.yml +++ b/.github/filters/integration-tests.yml @@ -6,10 +6,6 @@ dbt-athena: - 'dbt-adapters/pyproject.toml' - 'dbt-athena/tests/functional/**' - 'dbt-athena/tests/__init__.py' -dbt-athena-community: -- 'dbt-athena-community/src/**' -- 'dbt-athena-community/hatch.toml' -- 'dbt-athena-community/pyproject.toml' dbt-bigquery: - 'dbt-bigquery/src/**' - 'dbt-bigquery/hatch.toml' diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml index ae7defad2..b48e4f0d9 100644 --- a/.github/workflows/_integration-tests.yml +++ b/.github/workflows/_integration-tests.yml @@ -3,8 +3,8 @@ name: "Integration tests" on: workflow_call: inputs: - package: - description: "Choose the package to test" + packages: + description: "List the packages to test as a JSON string" type: string required: true branch: @@ -25,17 +25,10 @@ on: default: "3.9" workflow_dispatch: inputs: - package: - description: "Choose the package to test" - type: choice - options: - - "dbt-athena" - - "dbt-athena-community" - - "dbt-bigquery" - - "dbt-postgres" - - "dbt-redshift" - - "dbt-snowflake" - - "dbt-spark" + packages: + description: "List the packages to test as a JSON string" + type: string + required: true branch: description: "Choose the branch to test" type: string @@ -70,8 +63,11 @@ env: jobs: integration-tests-athena: - if: ${{ inputs.package == 'dbt-athena' || inputs.package == 'dbt-athena-community' }} + if: contains(fromJSON(inputs.packages), 'dbt-athena') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-athena" environment: name: "dbt-athena" env: @@ -98,11 +94,13 @@ jobs: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }} aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} - run: hatch run integration-tests - working-directory: ./${{ inputs.package }} integration-tests-bigquery: - if: ${{ inputs.package == 'dbt-bigquery' }} + if: contains(fromJSON(inputs.packages), 'dbt-bigquery') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-bigquery" environment: name: "dbt-bigquery" env: @@ -125,12 +123,16 @@ jobs: python-version: ${{ inputs.python-version }} - uses: pypa/hatch@install - run: hatch run integration-tests tests/functional -k "not TestPython" - working-directory: ./${{ inputs.package }} integration-tests-bigquery-flaky: # we only run this for one python version to avoid running in parallel - if: ${{ inputs.package == 'dbt-bigquery' && inputs.python-version == vars.DEFAULT_PYTHON_VERSION }} + if: | + contains(fromJSON(inputs.packages), 'dbt-bigquery') && + inputs.python-version == vars.DEFAULT_PYTHON_VERSION runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-bigquery" environment: name: "dbt-bigquery" env: @@ -153,13 +155,15 @@ jobs: python-version: ${{ inputs.python-version }} - uses: pypa/hatch@install - run: hatch run integration-tests tests/functional -n1 -k "TestPython" - working-directory: ./${{ inputs.package }} integration-tests-postgres: - if: inputs.package == 'dbt-postgres' + if: contains(fromJSON(inputs.packages), 'dbt-postgres') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-postgres" environment: - name: ${{ inputs.package }} + name: "dbt-postgres" env: POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }} POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }} @@ -188,7 +192,6 @@ jobs: with: python-version: ${{ inputs.python-version }} - run: psql -f ./scripts/setup_test_database.sql - working-directory: ./dbt-postgres env: PGHOST: ${{ vars.POSTGRES_TEST_HOST }} PGPORT: ${{ vars.POSTGRES_TEST_PORT }} @@ -197,11 +200,13 @@ jobs: PGDATABASE: postgres - uses: pypa/hatch@install - run: hatch run integration-tests tests/functional --ddtrace - working-directory: ./${{ inputs.package }} integration-tests-postgres-psycopg2-check: - if: inputs.package == 'dbt-postgres' + if: contains(fromJSON(inputs.packages), 'dbt-postgres') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-postgres" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -211,32 +216,31 @@ jobs: run: brew install postgresql - name: "Test psycopg2 - default" run: scripts/psycopg2-check.sh - working-directory: ./${{ inputs.package }} env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 - invalid override" run: scripts/psycopg2-check.sh - working-directory: ./${{ inputs.package }} env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 - override" run: scripts/psycopg2-check.sh - working-directory: ./${{ inputs.package }} env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work - name: "Test psycopg2 - manual override" # verify that the workaround documented in the `README.md` continues to work run: scripts/psycopg2-check.sh - working-directory: ./${{ inputs.package }} env: PSYCOPG2_WORKAROUND: true PSYCOPG2_EXPECTED_NAME: psycopg2 integration-tests-redshift: - if: ${{ inputs.package == 'dbt-redshift' }} + if: contains(fromJSON(inputs.packages), 'dbt-redshift') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-redshift" environment: name: "dbt-redshift" env: @@ -286,12 +290,16 @@ jobs: aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION aws configure --profile $AWS_ROLE_PROFILE set output json - run: hatch run integration-tests tests/functional -m "not flaky" --ddtrace - working-directory: ./${{ inputs.package }} integration-tests-redshift-flaky: # we only run this for one python version to avoid running in parallel - if: ${{ inputs.package == 'dbt-redshift' && inputs.python-version == vars.DEFAULT_PYTHON_VERSION }} + if: | + contains(fromJSON(inputs.packages), 'dbt-redshift') && + inputs.python-version == vars.DEFAULT_PYTHON_VERSION runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-redshift" environment: name: "dbt-redshift" env: @@ -341,13 +349,15 @@ jobs: aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION aws configure --profile $AWS_ROLE_PROFILE set output json - run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace - working-directory: ./${{ inputs.package }} integration-tests-snowflake: - if: ${{ inputs.package == 'dbt-snowflake' }} + if: contains(fromJSON(inputs.packages), 'dbt-snowflake') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-snowflake" environment: - name: ${{ inputs.package }} + name: "dbt-snowflake" env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }} @@ -373,11 +383,13 @@ jobs: python-version: ${{ inputs.python-version }} - uses: pypa/hatch@install - run: hatch run integration-tests tests/functional --ddtrace - working-directory: ./${{ inputs.package }} integration-tests-spark: - if: ${{ inputs.package == 'dbt-spark' }} + if: contains(fromJSON(inputs.packages), 'dbt-spark') runs-on: ${{ inputs.os }} + defaults: + run: + working-directory: "./dbt-spark" environment: name: "dbt-spark" env: @@ -406,6 +418,4 @@ jobs: python-version: ${{ inputs.python-version }} - uses: pypa/hatch@install - run: hatch run pip install -r dagger/requirements.txt - working-directory: ./${{ inputs.package }} - run: hatch run integration-tests --profile ${{ matrix.profile }} - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f0bf8df8e..1bf58d34c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -67,7 +67,7 @@ jobs: !contains(fromJSON('["dbt-adapters", "dbt-tests-adapter"]'), inputs.package) uses: ./.github/workflows/_integration-tests.yml with: - package: ${{ inputs.package }} + packages: ${{ toJSON(inputs.package) }} branch: ${{ inputs.branch }} secrets: inherit diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index b4e224e45..e1b5ebaee 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -47,7 +47,7 @@ jobs: changelog-entry-check: uses: ./.github/workflows/_changelog-entry-check.yml needs: affected-packages - if: ${{ toJson(fromJson(needs.affected-packages.outputs.changelog-entry-check)) != '[]' }} + if: ${{ toJSON(fromJSON(needs.affected-packages.outputs.changelog-entry-check)) != '[]' }} strategy: fail-fast: false matrix: @@ -66,7 +66,7 @@ jobs: verify-build: uses: ./.github/workflows/_verify-build.yml needs: affected-packages - if: ${{ toJson(fromJson(needs.affected-packages.outputs.verify-build)) != '[]' }} + if: ${{ toJSON(fromJSON(needs.affected-packages.outputs.verify-build)) != '[]' }} strategy: fail-fast: false matrix: @@ -81,7 +81,7 @@ jobs: unit-tests: uses: ./.github/workflows/_unit-tests.yml needs: affected-packages - if: ${{ toJson(fromJson(needs.affected-packages.outputs.unit-tests)) != '[]' }} + if: ${{ toJSON(fromJSON(needs.affected-packages.outputs.unit-tests)) != '[]' }} strategy: fail-fast: false matrix: @@ -96,13 +96,9 @@ jobs: integration-tests: uses: ./.github/workflows/_integration-tests.yml needs: affected-packages - if: ${{ toJson(fromJson(needs.affected-packages.outputs.integration-tests)) != '[]' }} - strategy: - fail-fast: false - matrix: - package: ${{ fromJSON(needs.affected-packages.outputs.integration-tests) }} + if: ${{ toJSON(fromJSON(needs.affected-packages.outputs.integration-tests)) != '[]' }} with: - package: ${{ matrix.package }} + packages: ${{ fromJSON(needs.affected-packages.outputs.integration-tests) }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} os: ${{ vars.DEFAULT_RUNNER }} diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml index 21d1dab39..0fa2cb98c 100644 --- a/.github/workflows/scheduled-tests.yml +++ b/.github/workflows/scheduled-tests.yml @@ -22,7 +22,7 @@ jobs: - "dbt-snowflake" - "dbt-spark" os: [ubuntu-22.04, macos-14, windows-2022] - python-version: ${{ fromJson(vars.SUPPORTED_PYTHON_VERSIONS) }} + python-version: ${{ fromJSON(vars.SUPPORTED_PYTHON_VERSIONS) }} with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} @@ -45,7 +45,7 @@ jobs: - "dbt-snowflake" - "dbt-spark" os: [ubuntu-22.04, macos-14, windows-2022] - python-version: ${{ fromJson(vars.SUPPORTED_PYTHON_VERSIONS) }} + python-version: ${{ fromJSON(vars.SUPPORTED_PYTHON_VERSIONS) }} with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} @@ -55,19 +55,8 @@ jobs: integration-tests: uses: ./.github/workflows/_integration-tests.yml - strategy: - fail-fast: false - matrix: - package: - - "dbt-athena" - - "dbt-athena-community" - - "dbt-bigquery" - - "dbt-postgres" - - "dbt-redshift" - - "dbt-snowflake" - - "dbt-spark" with: - package: ${{ matrix.package }} + packages: ${{ toJSON('[dbt-athena, dbt-bigquery, dbt-postgres, dbt-redshift, dbt-snowflake, dbt-spark]') }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} os: ${{ vars.DEFAULT_RUNNER }}