diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index c7f787716..7852bb735 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -41,6 +41,8 @@ body: required: false - label: dbt-redshift required: false + - label: dbt-snowflake + required: false - label: dbt-spark required: false - type: textarea diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 95264bd9e..5214dac7d 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -36,6 +36,8 @@ body: required: false - label: dbt-redshift required: false + - label: dbt-snowflake + required: false - label: dbt-spark required: false - type: textarea diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 331886dfa..abca115de 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,7 @@ updates: - "/dbt-bigquery" - "/dbt-postgres" - "/dbt-redshift" + - "/dbt-snowflake" - "/dbt-spark" schedule: interval: "daily" @@ -22,6 +23,7 @@ updates: - "/dbt-bigquery/docker" - "/dbt-postgres/docker" - "/dbt-redshift/docker" + - "/dbt-snowflake/docker" - "/dbt-spark/docker" schedule: interval: "weekly" diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index 86bce56cc..b2c62010c 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -38,6 +38,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" merge: description: "Choose whether to merge the changelog branch" diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml index 9410fc678..4a74fa650 100644 --- a/.github/workflows/_integration-tests.yml +++ b/.github/workflows/_integration-tests.yml @@ -34,6 +34,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" branch: description: "Choose the branch to test" @@ -267,6 +268,38 @@ jobs: - run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace working-directory: ./${{ inputs.package }} + integration-tests-snowflake: + if: ${{ inputs.package == 'dbt-snowflake' }} + runs-on: ${{ inputs.os }} + environment: + name: ${{ inputs.package }} + env: + SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} + SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }} + SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} + SNOWFLAKE_TEST_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }} + SNOWFLAKE_TEST_DATABASE: ${{ vars.SNOWFLAKE_TEST_DATABASE }} + SNOWFLAKE_TEST_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WAREHOUSE }} + SNOWFLAKE_TEST_ALT_DATABASE: ${{ vars.SNOWFLAKE_TEST_ALT_DATABASE }} + SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_ALT_WAREHOUSE }} + SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ vars.SNOWFLAKE_TEST_QUOTED_DATABASE }} + SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ vars.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }} + SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }} + SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }} + SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY }} + SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - run: hatch run integration-tests tests/functional --ddtrace + working-directory: ./${{ inputs.package }} + integration-tests-spark: if: ${{ inputs.package == 'dbt-spark' }} runs-on: ${{ inputs.os }} diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index f59a64c96..c45e1be3b 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -26,6 +26,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" deploy-to: description: "Choose whether to publish to test or prod" diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml index 9b7abaf26..f8bd4156d 100644 --- a/.github/workflows/_publish-pypi.yml +++ b/.github/workflows/_publish-pypi.yml @@ -27,6 +27,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" deploy-to: description: "Choose whether to publish to test or prod" diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index 121eb170b..f151371e7 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -35,6 +35,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" branch: description: "Choose the branch to test" diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index 4ce2a1019..6b01fa29e 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -36,6 +36,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" branch: description: "Choose the branch to build" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4a72927ab..9674430f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,6 +15,7 @@ on: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" deploy-to: description: "Choose whether to publish to test or prod" diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 3b923826c..4250ed1e3 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -37,6 +37,7 @@ jobs: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" os: [ubuntu-22.04] python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -59,6 +60,7 @@ jobs: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" os: [ ubuntu-22.04 ] python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -80,6 +82,7 @@ jobs: - "dbt-bigquery" - "dbt-postgres" - "dbt-redshift" + - "dbt-snowflake" - "dbt-spark" os: [ubuntu-22.04] python-version: ["3.9", "3.10", "3.11", "3.12"] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de7e2c63a..536b25715 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: 7.1.1 hooks: - id: flake8 - exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests + exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests|dbt-snowflake/tests args: - --max-line-length=99 - --select=E,F,W diff --git a/dbt-snowflake/.changes/0.0.0.md b/dbt-snowflake/.changes/0.0.0.md new file mode 100644 index 000000000..23c94e2b9 --- /dev/null +++ b/dbt-snowflake/.changes/0.0.0.md @@ -0,0 +1,9 @@ +## Previous Releases +For information on prior major and minor releases, see their changelogs: +- [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md) +- [1.5](https://github.com/dbt-labs/dbt-snowflake/blob/1.5.latest/CHANGELOG.md) +- [1.4](https://github.com/dbt-labs/dbt-snowflake/blob/1.4.latest/CHANGELOG.md) +- [1.3](https://github.com/dbt-labs/dbt-snowflake/blob/1.3.latest/CHANGELOG.md) +- [1.2](https://github.com/dbt-labs/dbt-snowflake/blob/1.2.latest/CHANGELOG.md) +- [1.1](https://github.com/dbt-labs/dbt-snowflake/blob/1.1.latest/CHANGELOG.md) +- [1.0](https://github.com/dbt-labs/dbt-snowflake/blob/1.0.latest/CHANGELOG.md) diff --git a/dbt-snowflake/.changes/README.md b/dbt-snowflake/.changes/README.md new file mode 100644 index 000000000..4166515d7 --- /dev/null +++ b/dbt-snowflake/.changes/README.md @@ -0,0 +1,4 @@ +# CHANGELOG + + +To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-snowflake/blob/main/.changes/README.md) found in `dbt-snowflake`. diff --git a/dbt-snowflake/.changes/header.tpl.md b/dbt-snowflake/.changes/header.tpl.md new file mode 100644 index 000000000..1022a1999 --- /dev/null +++ b/dbt-snowflake/.changes/header.tpl.md @@ -0,0 +1,6 @@ +# dbt-snowflake Changelog + +- This file provides a full account of all changes to `dbt-snowflake`. +- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. +- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. +- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) diff --git a/dbt-snowflake/.changes/unreleased/.gitkeep b/dbt-snowflake/.changes/unreleased/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/.changes/unreleased/Features-20241202-095136.yaml b/dbt-snowflake/.changes/unreleased/Features-20241202-095136.yaml new file mode 100644 index 000000000..973866a6d --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Features-20241202-095136.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support MicrobatchConcurrency +time: 2024-12-02T09:51:36.606097-05:00 +custom: + Author: michelleark + Issue: "1260" diff --git a/dbt-snowflake/.changes/unreleased/Features-20250113-133414.yaml b/dbt-snowflake/.changes/unreleased/Features-20250113-133414.yaml new file mode 100644 index 000000000..869ed6b17 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Features-20250113-133414.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Added support for custom iceberg base_location_root +time: 2025-01-13T13:34:14.326047-08:00 +custom: + Author: LProcopi15 + Issue: "1284" diff --git a/dbt-snowflake/.changes/unreleased/Fixes-20241018-173123.yaml b/dbt-snowflake/.changes/unreleased/Fixes-20241018-173123.yaml new file mode 100644 index 000000000..eab4e8376 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Fixes-20241018-173123.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix parsing of the VECTOR type +time: 2024-10-18T17:31:23.931299-04:00 +custom: + Author: achawkins + Issue: "1098" diff --git a/dbt-snowflake/.changes/unreleased/Fixes-20241127-162204.yaml b/dbt-snowflake/.changes/unreleased/Fixes-20241127-162204.yaml new file mode 100644 index 000000000..2b990b1f9 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Fixes-20241127-162204.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use timestamp_tz type in microbatch `delete` DDL +time: 2024-11-27T16:22:04.103212-05:00 +custom: + Author: michelleark + Issue: "1256" diff --git a/dbt-snowflake/.changes/unreleased/Fixes-20241209-131530.yaml b/dbt-snowflake/.changes/unreleased/Fixes-20241209-131530.yaml new file mode 100644 index 000000000..b3196a3b0 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Fixes-20241209-131530.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: AUTO should no longer lead to rebuilds of dynamic tables. +time: 2024-12-09T13:15:30.554566-08:00 +custom: + Author: versusfacit + Issue: "1267" diff --git a/dbt-snowflake/.changes/unreleased/Under the Hood-20241117-184430.yaml b/dbt-snowflake/.changes/unreleased/Under the Hood-20241117-184430.yaml new file mode 100644 index 000000000..d2a7b67a6 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Under the Hood-20241117-184430.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml and to hatch as a dev tool +time: 2024-11-17T18:44:30.85288-05:00 +custom: + Author: mikealfare + Issue: "1250" diff --git a/dbt-snowflake/.changes/unreleased/Under the Hood-20241205-144036.yaml b/dbt-snowflake/.changes/unreleased/Under the Hood-20241205-144036.yaml new file mode 100644 index 000000000..aedcb4ce1 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Under the Hood-20241205-144036.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Use new `batch` context variables over `node.config.__dbt_internal` ones +time: 2024-12-05T14:40:36.373637-05:00 +custom: + Author: michelleark + Issue: "1263" diff --git a/dbt-snowflake/.changes/unreleased/Under the Hood-20241211-170831.yaml b/dbt-snowflake/.changes/unreleased/Under the Hood-20241211-170831.yaml new file mode 100644 index 000000000..17bf42c1f --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Under the Hood-20241211-170831.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update default Python version for python models +time: 2024-12-11T17:08:31.842063-05:00 +custom: + Author: mikealfare + Issue: "1203" diff --git a/dbt-snowflake/.changie.yaml b/dbt-snowflake/.changie.yaml new file mode 100644 index 000000000..6fe1c8392 --- /dev/null +++ b/dbt-snowflake/.changie.yaml @@ -0,0 +1,132 @@ +changesDir: .changes +unreleasedDir: unreleased +headerPath: header.tpl.md +versionHeaderPath: "" +changelogPath: CHANGELOG.md +versionExt: md +envPrefix: "CHANGIE_" +versionFormat: '## dbt-snowflake {{.Version}} - {{.Time.Format "January 02, 2006"}}' +kindFormat: '### {{.Kind}}' +changeFormat: |- + {{- $IssueList := list }} + {{- $changes := splitList " " $.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) + +kinds: +- label: Breaking Changes +- label: Features +- label: Fixes +- label: Under the Hood +- label: Dependencies + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +- label: Security + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 + +newlines: + afterChangelogHeader: 1 + afterKind: 1 + afterChangelogVersion: 1 + beforeKind: 1 + endOfVersion: 1 + +custom: +- key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 +- key: Issue + label: GitHub Issue Number (separated by a single space if multiple) + type: string + minLength: 1 + + +footerFormat: | + {{- $contributorDict := dict }} + {{- /* ensure all names in this list are all lowercase for later matching purposes */}} + {{- $core_team := splitList " " .Env.CORE_TEAM }} + {{- /* ensure we always skip snyk and dependabot in addition to the core team */}} + {{- $maintainers := list "dependabot[bot]" "snyk-bot"}} + {{- range $team_member := $core_team }} + {{- $team_member_lower := lower $team_member }} + {{- $maintainers = append $maintainers $team_member_lower }} + {{- end }} + {{- range $change := .Changes }} + {{- $authorList := splitList " " $change.Custom.Author }} + {{- /* loop through all authors for a single changelog */}} + {{- range $author := $authorList }} + {{- $authorLower := lower $author }} + {{- /* we only want to include non-core team contributors */}} + {{- if not (has $authorLower $maintainers)}} + {{- $changeList := splitList " " $change.Custom.Author }} + {{- $IssueList := list }} + {{- $changeLink := $change.Kind }} + {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} + {{- $changes := splitList " " $change.Custom.PR }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- else }} + {{- $changes := splitList " " $change.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- end }} + {{- /* check if this contributor has other changes associated with them already */}} + {{- if hasKey $contributorDict $author }} + {{- $contributionList := get $contributorDict $author }} + {{- $contributionList = concat $contributionList $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- else }} + {{- $contributionList := $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- end }} + {{- end}} + {{- end}} + {{- end }} + {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} + {{- if $contributorDict}} + ### Contributors + {{- range $k,$v := $contributorDict }} + - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) + {{- end }} + {{- end }} diff --git a/dbt-snowflake/CHANGELOG.md b/dbt-snowflake/CHANGELOG.md new file mode 100644 index 000000000..8b6702b8f --- /dev/null +++ b/dbt-snowflake/CHANGELOG.md @@ -0,0 +1,16 @@ +# dbt-snowflake Changelog + +- This file provides a full account of all changes to `dbt-snowflake`. +- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. +- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. +- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) + +## Previous Releases +For information on prior major and minor releases, see their changelogs: +- [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md) +- [1.5](https://github.com/dbt-labs/dbt-snowflake/blob/1.5.latest/CHANGELOG.md) +- [1.4](https://github.com/dbt-labs/dbt-snowflake/blob/1.4.latest/CHANGELOG.md) +- [1.3](https://github.com/dbt-labs/dbt-snowflake/blob/1.3.latest/CHANGELOG.md) +- [1.2](https://github.com/dbt-labs/dbt-snowflake/blob/1.2.latest/CHANGELOG.md) +- [1.1](https://github.com/dbt-labs/dbt-snowflake/blob/1.1.latest/CHANGELOG.md) +- [1.0](https://github.com/dbt-labs/dbt-snowflake/blob/1.0.latest/CHANGELOG.md) diff --git a/dbt-snowflake/CONTRIBUTING.md b/dbt-snowflake/CONTRIBUTING.md new file mode 100644 index 000000000..5b68aa03a --- /dev/null +++ b/dbt-snowflake/CONTRIBUTING.md @@ -0,0 +1,128 @@ +# Contributing to `dbt-snowflake` + +1. [About this document](#about-this-document) +3. [Getting the code](#getting-the-code) +5. [Running `dbt-snowflake` in development](#running-dbt-snowflake-in-development) +6. [Testing](#testing) +7. [Updating Docs](#updating-docs) +7. [Submitting a Pull Request](#submitting-a-pull-request) + +## About this document +This document is a guide for anyone interested in contributing to the `dbt-snowflake` repository. It outlines how to create issues and submit pull requests (PRs). + +This is not intended as a guide for using `dbt-snowflake` in a project. For configuring and using this adapter, see [Snowflake Profile](https://docs.getdbt.com/reference/warehouse-profiles/snowflake-profile), and [Snowflake Configs](https://docs.getdbt.com/reference/resource-configs/snowflake-configs). + +We assume users have a Linux or MacOS system. You should have familiarity with: + +- Python `virturalenv`s +- Python modules +- `pip` +- common command line utilities like `git`. + +In addition to this guide, we highly encourage you to read the [dbt-core](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md). Almost all information there is applicable here! + +### Signing the CLA + +Please note that all contributors to `dbt-snowflake` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA) before their pull request(s) can be merged into the `dbt-snowflake` codebase. Given this, `dbt-snowflake` maintainers will unfortunately be unable to merge your contribution(s) until you've signed the CLA. You are, however, welcome to open issues and comment on existing ones. + +## Getting the code + + `git` is needed in order to download and modify the `dbt-snowflake` code. There are several ways to install Git. For MacOS, we suggest installing [Xcode](https://developer.apple.com/support/xcode/) or [Xcode Command Line Tools](https://mac.install.guide/commandlinetools/index.html). + +### External contributors + +If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-snowflake` by forking the `dbt-snowflake` repository. For more on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: + +1. fork the `dbt-snowflake` repository +2. clone your fork locally +3. check out a new branch for your proposed changes +4. push changes to your fork +5. open a pull request of your forked repository against `dbt-labs/dbt-snowflake` + +### dbt Labs contributors + +If you are a member of the `dbt Labs` GitHub organization, you will have push access to the `dbt-snowflake` repo. Rather than forking `dbt-snowflake` to make your changes, clone the repository like normal, and check out feature branches. + +## Running `dbt-snowflake` in development + +### Installation + +1. Ensure you have the latest version of `pip` installed by running `pip install --upgrade pip` in terminal. + +2. Configure and activate a `virtualenv` as described in [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment). + +3. Install `dbt-core` in the active `virtualenv`. To confirm you installed dbt correctly, run `dbt --version` and `which dbt`. + +4. Install `dbt-snowflake` and development dependencies in the active `virtualenv`. Run `pip install -e . -r dev-requirements.txt`. + +When `dbt-snowflake` is installed this way, any changes you make to the `dbt-snowflake` source code will be reflected immediately (i.e. in your next local dbt invocation against a Snowflake target). + +## Testing + +### Initial setup + +`dbt-snowflake` contains [unit](https://github.com/dbt-labs/dbt-snowflake/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-snowflake/tree/main/tests/functional) tests. Functional tests require an actual Snowflake warehouse to test against. There are two primary ways to do this: + +- This repo has CI/CD GitHub Actions set up. Both unit and functional tests will run against an already configured Snowflake warehouse during PR checks. + +- You can also run functional tests "locally" by configuring a `test.env` file with appropriate `ENV` variables. + +``` +cp test.env.example test.env +$EDITOR test.env +``` + +WARNING: The parameters in your `test.env` file must link to a valid Snowflake account. The `test.env` file you create is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. + + +### "Local" test commands +There are a few methods for running tests locally. + +#### `tox` +`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`. + +#### `pytest` +You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so: + +```sh +# Note: replace $strings with valid names + +# run all snowflake functional tests in a directory +python -m pytest tests/functional/$test_directory +# run all snowflake functional tests in a module +python -m pytest -m profile_snowflake tests/functional/$test_dir_and_filename.py +# run all snowflake functional tests in a class +python -m pytest -m profile_snowflake tests/functional/$test_dir_and_filename.py::$test_class_name +# run a specific snowflake functional test +python -m pytest -m profile_snowflake tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name + +# run all unit tests in a module +python -m pytest tests/unit/$test_file_name.py +# run a specific unit test +python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name +``` + +## Updating documentation + +Many changes will require an update to `dbt-snowflake` documentation. Here are some relevant links. + +- Docs are [here](https://docs.getdbt.com/). +- The docs repo for making changes is located [here](https://github.com/dbt-labs/docs.getdbt.com). +- The changes made are likely to impact one or both of [Snowflake Profile](https://docs.getdbt.com/reference/warehouse-profiles/snowflake-profile), or [Snowflake Configs](https://docs.getdbt.com/reference/resource-configs/snowflake-configs). +- We ask every community member who makes a user-facing change to open an issue or PR regarding doc changes. + +## Adding CHANGELOG Entry + +We use [changie](https://changie.dev) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost. + +Follow the steps to [install `changie`](https://changie.dev/guide/installation/) for your system. + +Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete! + +You don't need to worry about which `dbt-snowflake` version your change will go into. Just create the changelog entry with `changie`, and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-snowflake`. The Core maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-snowflake`. + +## Submitting a Pull Request + +A `dbt-snowflake` maintainer will review your PR and will determine if it has passed regression tests. They may suggest code revisions for style and clarity, or they may request that you add unit or functional tests. These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. + +Once all tests are passing and your PR has been approved, a `dbt-snowflake` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: diff --git a/dbt-snowflake/LICENSE.md b/dbt-snowflake/LICENSE.md new file mode 100644 index 000000000..536bebee0 --- /dev/null +++ b/dbt-snowflake/LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 dbt Labs, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dbt-snowflake/README.md b/dbt-snowflake/README.md new file mode 100644 index 000000000..05beb0b40 --- /dev/null +++ b/dbt-snowflake/README.md @@ -0,0 +1,39 @@ +

+ dbt logo +

+

+ + Unit Tests Badge + + + Integration Tests Badge + +

+ +**[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. + +dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. + +## dbt-snowflake + +The `dbt-snowflake` package contains all of the code enabling dbt to work with Snowflake. For +more information on using dbt with Snowflake, consult [the docs](https://docs.getdbt.com/docs/profile-snowflake). + +## Getting started + +- [Install dbt](https://docs.getdbt.com/docs/installation) +- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) + +## Join the dbt Community + +- Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) +- Read more on the [dbt Community Discourse](https://discourse.getdbt.com) + +## Reporting bugs and contributing code + +- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt-snowflake/issues/new) +- Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md) + +## Code of Conduct + +Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). diff --git a/dbt-snowflake/docker/Dockerfile b/dbt-snowflake/docker/Dockerfile new file mode 100644 index 000000000..16060db61 --- /dev/null +++ b/dbt-snowflake/docker/Dockerfile @@ -0,0 +1,37 @@ +# this image gets published to GHCR for production use +ARG py_version=3.11.2 + +FROM python:$py_version-slim-bullseye AS base + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.18-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base AS dbt-snowflake + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/dbt-snowflake@${commit_ref}" diff --git a/dbt-snowflake/docker/README.md b/dbt-snowflake/docker/README.md new file mode 100644 index 000000000..95ecde101 --- /dev/null +++ b/dbt-snowflake/docker/README.md @@ -0,0 +1,58 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-snowflake` + +In order to build a new image, run the following docker command. +```shell +docker build --tag --target dbt-snowflake +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the image will be populated with the latest version of `dbt-snowflake` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +```shell +docker build --tag \ + --target dbt-snowflake \ + --build-arg commit_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +```shell +cd dbt-core/docker +docker build --tag my-dbt --target dbt-snowflake . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +```shell +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-snowflake \ + --build-arg commit_ref=v1.0.0b1 \ + . +``` + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +```shell +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/dbt-snowflake/docker/dev.Dockerfile b/dbt-snowflake/docker/dev.Dockerfile new file mode 100644 index 000000000..44f86f005 --- /dev/null +++ b/dbt-snowflake/docker/dev.Dockerfile @@ -0,0 +1,50 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:24.04 AS base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.48 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.10ubuntu1 \ + git-all=1:2.43.0-1ubuntu7.1 \ + python3.9=3.9.20-1+noble1 \ + python3.9-dev=3.9.20-1+noble1 \ + python3.9-distutils=3.9.20-1+noble1 \ + python3.9-venv=3.9.20-1+noble1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 + + +FROM base AS dbt-snowflake-dev + +HEALTHCHECK CMD python --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python -m venv /opt/venv diff --git a/dbt-snowflake/docker/test.sh b/dbt-snowflake/docker/test.sh new file mode 100755 index 000000000..79311b6c9 --- /dev/null +++ b/dbt-snowflake/docker/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-snowflake latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-snowflake \ + --target dbt-snowflake \ + docker \ +&& docker run dbt-snowflake --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-snowflake-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-snowflake-1.0.0b1 \ + --target dbt-snowflake \ + --build-arg commit_ref=v1.0.0b1 \ + docker \ +&& docker run dbt-snowflake-1.0.0b1 --version diff --git a/dbt-snowflake/hatch.toml b/dbt-snowflake/hatch.toml new file mode 100644 index 000000000..7a2f1e272 --- /dev/null +++ b/dbt-snowflake/hatch.toml @@ -0,0 +1,62 @@ +[version] +path = "src/dbt/adapters/snowflake/__version__.py" + +[build.targets.sdist] +packages = ["src/dbt"] +sources = ["src"] + +[build.targets.wheel] +packages = ["src/dbt"] +sources = ["src"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "ddtrace==2.3.0", + "ipdb~=0.13.13", + "pre-commit~=3.7.0", + "pytest~=7.4", + "pytest-csv~=3.0", + "pytest-dotenv~=0.5.2", + "pytest-logbook~=1.2", + "pytest-xdist~=3.6", + "tox~=4.16", # does this pin deps transitively? +] + +[envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "- python -m pytest {args:tests/functional}" +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", + "docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] + +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt_snowflake-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_snowflake-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ." diff --git a/dbt-snowflake/pyproject.toml b/dbt-snowflake/pyproject.toml new file mode 100644 index 000000000..568aa3533 --- /dev/null +++ b/dbt-snowflake/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-snowflake" +description = "The Snowflake adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"] +requires-python = ">=3.9.0" +authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.10.4,<2.0", + "snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0", + # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "agate", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-snowflake" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-snowflake.git" +Issues = "https://github.com/dbt-labs/dbt-snowflake/issues" +Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" + +[tool.pytest.ini_options] +testpaths = ["tests/functional", "tests/unit"] +env_files = ["test.env"] +addopts = "-v --color=yes -n auto" +filterwarnings = [ + "ignore:datetime.datetime.utcnow:DeprecationWarning", +] diff --git a/dbt-snowflake/scripts/build-dist.sh b/dbt-snowflake/scripts/build-dist.sh new file mode 100755 index 000000000..3c3808399 --- /dev/null +++ b/dbt-snowflake/scripts/build-dist.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" + +PYTHON_BIN=${PYTHON_BIN:-python} + +echo "$PYTHON_BIN" + +set -x + +rm -rf "$DBT_PATH"/dist +rm -rf "$DBT_PATH"/build +mkdir -p "$DBT_PATH"/dist + +cd "$DBT_PATH" +$PYTHON_BIN setup.py sdist bdist_wheel + +set +x diff --git a/dbt-snowflake/scripts/env-setup.sh b/dbt-snowflake/scripts/env-setup.sh new file mode 100644 index 000000000..f05c705ce --- /dev/null +++ b/dbt-snowflake/scripts/env-setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Set TOXENV environment variable for subsequent steps +echo "TOXENV=integration-snowflake" >> $GITHUB_ENV +# Set INTEGRATION_TESTS_SECRETS_PREFIX environment variable for subsequent steps +# All GH secrets that have this prefix will be set as environment variables +echo "INTEGRATION_TESTS_SECRETS_PREFIX=SNOWFLAKE_TEST" >> $GITHUB_ENV +# Set environment variables required for integration tests +echo "DBT_TEST_USER_1=dbt_test_role_1" >> $GITHUB_ENV +echo "DBT_TEST_USER_2=dbt_test_role_2" >> $GITHUB_ENV +echo "DBT_TEST_USER_3=dbt_test_role_3" >> $GITHUB_ENV diff --git a/dbt-snowflake/scripts/update_dev_dependency_branches.sh b/dbt-snowflake/scripts/update_dev_dependency_branches.sh new file mode 100755 index 000000000..603dec17b --- /dev/null +++ b/dbt-snowflake/scripts/update_dev_dependency_branches.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e +set -e + + +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${dbt_adapters_branch}#subdirectory=dbt-tests-adapter|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$tests_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file +else + sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$tests_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file +fi diff --git a/dbt-snowflake/scripts/update_release_branch.sh b/dbt-snowflake/scripts/update_release_branch.sh new file mode 100644 index 000000000..75b9ccef6 --- /dev/null +++ b/dbt-snowflake/scripts/update_release_branch.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e +set -e + +release_branch=$1 +target_req_file=".github/workflows/nightly-release.yml" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "s|[0-9].[0-9].latest|$release_branch|" $target_req_file +else + sed -i "s|[0-9].[0-9].latest|$release_branch|" $target_req_file +fi diff --git a/dbt-snowflake/scripts/werkzeug-refresh-token.py b/dbt-snowflake/scripts/werkzeug-refresh-token.py new file mode 100644 index 000000000..228ba2623 --- /dev/null +++ b/dbt-snowflake/scripts/werkzeug-refresh-token.py @@ -0,0 +1,144 @@ +import argparse +import json +import secrets +import textwrap +from base64 import b64encode + +import requests +from werkzeug.utils import redirect +from werkzeug.middleware.dispatcher import DispatcherMiddleware +from werkzeug.wrappers import Request, Response +from werkzeug.serving import run_simple +from urllib.parse import urlencode + + +def _make_rfp_claim_value(): + # from https://tools.ietf.org/id/draft-bradley-oauth-jwt-encoded-state-08.html#rfc.section.4 # noqa + # we can do whatever we want really, so just token.urlsafe? + return secrets.token_urlsafe(112) + + +def _make_response(client_id, client_secret, refresh_token): + return Response( + textwrap.dedent( + f'''\ + SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN="{refresh_token}" + SNOWFLAKE_TEST_OAUTH_CLIENT_ID="{client_id}" + SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET="{client_secret}"''' + ) + ) + + +class TokenManager: + def __init__(self, account_name, client_id, client_secret): + self.account_name = account_name + self.client_id = client_id + self.client_secret = client_secret + self.token = None + self.rfp_claim = _make_rfp_claim_value() + self.port = 8080 + + @property + def account_url(self): + return f"https://{self.account_name}.snowflakecomputing.com" + + @property + def auth_url(self): + return f"{self.account_url}/oauth/authorize" + + @property + def token_url(self): + return f"{self.account_url}/oauth/token-request" + + @property + def redirect_uri(self): + return f"http://localhost:{self.port}" + + @property + def headers(self): + auth = f"{self.client_id}:{self.client_secret}".encode("ascii") + encoded_auth = b64encode(auth).decode("ascii") + return { + "Authorization": f"Basic {encoded_auth}", + "Content-type": "application/x-www-form-urlencoded; charset=utf-8", + } + + def _code_to_token(self, code): + data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": self.redirect_uri, + } + # data = urlencode(data) + resp = requests.post( + url=self.token_url, + headers=self.headers, + data=data, + ) + try: + refresh_token = resp.json()["refresh_token"] + except KeyError: + print(resp.json()) + raise + return refresh_token + + @Request.application + def auth(self, request): + code = request.args.get("code") + if code: + # we got 303'ed here with a code + state_received = request.args.get("state") + if state_received != self.rfp_claim: + return Response("Invalid RFP claim: MITM?", status=401) + refresh_token = self._code_to_token(code) + return _make_response( + self.client_id, + self.client_secret, + refresh_token, + ) + else: + return redirect("/login") + + @Request.application + def login(self, request): + # take the auth URL and add the query string to it + query = { + "response_type": "code", + "client_id": self.client_id, + "redirect_uri": self.redirect_uri, + "state": self.rfp_claim, + } + query = urlencode(query) + return redirect(f"{self.auth_url}?{query}") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("account_name", help="The account name") + parser.add_argument("json_blob", help="The json auth blob") + + return parser.parse_args() + + +def main(): + args = parse_args() + data = json.loads(args.json_blob) + client_id = data["OAUTH_CLIENT_ID"] + client_secret = data["OAUTH_CLIENT_SECRET"] + token_manager = TokenManager( + account_name=args.account_name, + client_id=client_id, + client_secret=client_secret, + ) + app = DispatcherMiddleware( + token_manager.auth, + { + "/login": token_manager.login, + }, + ) + + run_simple("localhost", token_manager.port, app) + + +if __name__ == "__main__": + main() diff --git a/dbt-snowflake/src/dbt/__init__.py b/dbt-snowflake/src/dbt/__init__.py new file mode 100644 index 000000000..b36383a61 --- /dev/null +++ b/dbt-snowflake/src/dbt/__init__.py @@ -0,0 +1,3 @@ +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/__init__.py b/dbt-snowflake/src/dbt/adapters/snowflake/__init__.py new file mode 100644 index 000000000..f0c546067 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/__init__.py @@ -0,0 +1,12 @@ +from dbt.adapters.snowflake.column import SnowflakeColumn +from dbt.adapters.snowflake.connections import SnowflakeConnectionManager +from dbt.adapters.snowflake.connections import SnowflakeCredentials +from dbt.adapters.snowflake.relation import SnowflakeRelation +from dbt.adapters.snowflake.impl import SnowflakeAdapter + +from dbt.adapters.base import AdapterPlugin +from dbt.include import snowflake + +Plugin = AdapterPlugin( + adapter=SnowflakeAdapter, credentials=SnowflakeCredentials, include_path=snowflake.PACKAGE_PATH +) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/__version__.py b/dbt-snowflake/src/dbt/adapters/snowflake/__version__.py new file mode 100644 index 000000000..1af777a62 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/__version__.py @@ -0,0 +1 @@ +version = "1.10.0a1" diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/auth.py b/dbt-snowflake/src/dbt/adapters/snowflake/auth.py new file mode 100644 index 000000000..e914b6f3d --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/auth.py @@ -0,0 +1,57 @@ +import base64 +import sys +from typing import Optional + +if sys.version_info < (3, 9): + from functools import lru_cache + + cache = lru_cache(maxsize=None) +else: + from functools import cache + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey + + +@cache +def private_key_from_string( + private_key_string: str, passphrase: Optional[str] = None +) -> RSAPrivateKey: + + if passphrase: + encoded_passphrase = passphrase.encode() + else: + encoded_passphrase = None + + if private_key_string.startswith("-"): + return serialization.load_pem_private_key( + data=bytes(private_key_string, "utf-8"), + password=encoded_passphrase, + backend=default_backend(), + ) + return serialization.load_der_private_key( + data=base64.b64decode(private_key_string), + password=encoded_passphrase, + backend=default_backend(), + ) + + +@cache +def private_key_from_file( + private_key_path: str, passphrase: Optional[str] = None +) -> RSAPrivateKey: + + if passphrase: + encoded_passphrase = passphrase.encode() + else: + encoded_passphrase = None + + with open(private_key_path, "rb") as file: + private_key_bytes = file.read() + + return serialization.load_pem_private_key( + data=private_key_bytes, + password=encoded_passphrase, + backend=default_backend(), + ) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/column.py b/dbt-snowflake/src/dbt/adapters/snowflake/column.py new file mode 100644 index 000000000..281831b29 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/column.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass + +from dbt.adapters.base.column import Column +from dbt_common.exceptions import DbtRuntimeError + + +@dataclass +class SnowflakeColumn(Column): + def is_integer(self) -> bool: + # everything that smells like an int is actually a NUMBER(38, 0) + return False + + def is_numeric(self) -> bool: + return self.dtype.lower() in [ + "int", + "integer", + "bigint", + "smallint", + "tinyint", + "byteint", + "numeric", + "decimal", + "number", + ] + + def is_float(self): + return self.dtype.lower() in [ + "float", + "float4", + "float8", + "double", + "double precision", + "real", + ] + + def string_size(self) -> int: + if not self.is_string(): + raise DbtRuntimeError("Called string_size() on non-string field!") + + if self.dtype == "text" or self.char_size is None: + return 16777216 + else: + return int(self.char_size) + + @classmethod + def from_description(cls, name: str, raw_data_type: str) -> "SnowflakeColumn": + if "vector" in raw_data_type.lower(): + column = cls(name, raw_data_type, None, None, None) + else: + column = super().from_description(name, raw_data_type) + return column diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py new file mode 100644 index 000000000..fc2c09c19 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py @@ -0,0 +1,633 @@ +import base64 +import datetime +import os +import sys + +if sys.version_info < (3, 9): + from functools import lru_cache + + cache = lru_cache(maxsize=None) +else: + from functools import cache + +import pytz +import re +from contextlib import contextmanager +from dataclasses import dataclass +from io import StringIO +from time import sleep + +from typing import Optional, Tuple, Union, Any, List, Iterable, TYPE_CHECKING + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +import requests +import snowflake.connector +import snowflake.connector.constants +import snowflake.connector.errors +from snowflake.connector.errors import ( + Error, + DatabaseError, + InternalError, + InternalServerError, + ServiceUnavailableError, + GatewayTimeoutError, + RequestTimeoutError, + BadGatewayError, + OtherHTTPRetryableError, + BindUploadError, +) + +from dbt_common.exceptions import ( + DbtInternalError, + DbtRuntimeError, + DbtConfigError, +) +from dbt_common.exceptions import DbtDatabaseError +from dbt_common.record import get_record_mode_from_env, RecorderMode +from dbt.adapters.exceptions.connection import FailedToConnectError +from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials +from dbt.adapters.sql import SQLConnectionManager +from dbt.adapters.events.logging import AdapterLogger +from dbt_common.events.functions import warn_or_error +from dbt.adapters.events.types import AdapterEventWarning, AdapterEventError +from dbt_common.ui import line_wrap_message, warning_tag +from dbt.adapters.snowflake.record import SnowflakeRecordReplayHandle + +from dbt.adapters.snowflake.auth import private_key_from_file, private_key_from_string + +if TYPE_CHECKING: + import agate + + +logger = AdapterLogger("Snowflake") + +if os.getenv("DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING"): + for logger_name in ["snowflake.connector", "botocore", "boto3"]: + logger.debug(f"Setting {logger_name} to DEBUG") + logger.set_adapter_dependency_log_level(logger_name, "DEBUG") + +_TOKEN_REQUEST_URL = "https://{}.snowflakecomputing.com/oauth/token-request" + +ERROR_REDACTION_PATTERNS = { + re.compile(r"Row Values: \[(.|\n)*\]"): "Row Values: [redacted]", + re.compile(r"Duplicate field key '(.|\n)*'"): "Duplicate field key '[redacted]'", +} + + +@cache +def snowflake_private_key(private_key: RSAPrivateKey) -> bytes: + return private_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + +@dataclass +class SnowflakeCredentials(Credentials): + account: str + user: Optional[str] = None + warehouse: Optional[str] = None + role: Optional[str] = None + password: Optional[str] = None + authenticator: Optional[str] = None + private_key: Optional[str] = None + private_key_path: Optional[str] = None + private_key_passphrase: Optional[str] = None + token: Optional[str] = None + oauth_client_id: Optional[str] = None + oauth_client_secret: Optional[str] = None + query_tag: Optional[str] = None + client_session_keep_alive: bool = False + host: Optional[str] = None + port: Optional[int] = None + proxy_host: Optional[str] = None + proxy_port: Optional[int] = None + protocol: Optional[str] = None + connect_retries: int = 1 + connect_timeout: Optional[int] = None + retry_on_database_errors: bool = False + retry_all: bool = False + insecure_mode: Optional[bool] = False + # this needs to default to `None` so that we can tell if the user set it; see `__post_init__()` + reuse_connections: Optional[bool] = None + + def __post_init__(self): + if self.authenticator != "oauth" and (self.oauth_client_secret or self.oauth_client_id): + # the user probably forgot to set 'authenticator' like I keep doing + warn_or_error( + AdapterEventWarning( + base_msg="Authenticator is not set to oauth, but an oauth-only parameter is set! Did you mean to set authenticator: oauth?" + ) + ) + + if self.authenticator not in ["oauth", "jwt"]: + if self.token: + warn_or_error( + AdapterEventWarning( + base_msg=( + "The token parameter was set, but the authenticator was " + "not set to 'oauth' or 'jwt'." + ) + ) + ) + + if not self.user: + # The user attribute is only optional if 'authenticator' is 'jwt' or 'oauth' + warn_or_error( + AdapterEventError(base_msg="Invalid profile: 'user' is a required property.") + ) + + self.account = self.account.replace("_", "-") + + # only default `reuse_connections` to `True` if the user has not turned on `client_session_keep_alive` + # having both of these set to `True` could lead to hanging open connections, so it should be opt-in behavior + if self.client_session_keep_alive is False and self.reuse_connections is None: + self.reuse_connections = True + + @property + def type(self): + return "snowflake" + + @property + def unique_field(self): + return self.account + + # the results show up in the output of dbt debug runs, for more see.. + # https://docs.getdbt.com/guides/dbt-ecosystem/adapter-development/3-building-a-new-adapter#editing-the-connection-manager + def _connection_keys(self): + return ( + "account", + "user", + "database", + "warehouse", + "role", + "schema", + "authenticator", + "oauth_client_id", + "query_tag", + "client_session_keep_alive", + "host", + "port", + "proxy_host", + "proxy_port", + "protocol", + "connect_retries", + "connect_timeout", + "retry_on_database_errors", + "retry_all", + "insecure_mode", + "reuse_connections", + ) + + def auth_args(self): + # Pull all of the optional authentication args for the connector, + # let connector handle the actual arg validation + result = {} + if self.user: + result["user"] = self.user + if self.password: + result["password"] = self.password + if self.host: + result["host"] = self.host + if self.port: + result["port"] = self.port + if self.proxy_host: + result["proxy_host"] = self.proxy_host + if self.proxy_port: + result["proxy_port"] = self.proxy_port + if self.protocol: + result["protocol"] = self.protocol + if self.authenticator: + result["authenticator"] = self.authenticator + if self.authenticator == "oauth": + token = self.token + # if we have a client ID/client secret, the token is a refresh + # token, not an access token + if self.oauth_client_id and self.oauth_client_secret: + token = self._get_access_token() + elif self.oauth_client_id: + warn_or_error( + AdapterEventWarning( + base_msg="Invalid profile: got an oauth_client_id, but not an oauth_client_secret!" + ) + ) + elif self.oauth_client_secret: + warn_or_error( + AdapterEventWarning( + base_msg="Invalid profile: got an oauth_client_secret, but not an oauth_client_id!" + ) + ) + + result["token"] = token + + elif self.authenticator == "jwt": + # If authenticator is 'jwt', then the 'token' value should be used + # unmodified. We expose this as 'jwt' in the profile, but the value + # passed into the snowflake.connect method should still be 'oauth' + result["token"] = self.token + result["authenticator"] = "oauth" + + # enable id token cache for linux + result["client_store_temporary_credential"] = True + # enable mfa token cache for linux + result["client_request_mfa_token"] = True + result["reuse_connections"] = self.reuse_connections + result["private_key"] = self._get_private_key() + return result + + def _get_access_token(self) -> str: + if self.authenticator != "oauth": + raise DbtInternalError("Can only get access tokens for oauth") + missing = any( + x is None for x in (self.oauth_client_id, self.oauth_client_secret, self.token) + ) + if missing: + raise DbtInternalError( + "need a client ID a client secret, and a refresh token to get " "an access token" + ) + + # should the full url be a config item? + token_url = _TOKEN_REQUEST_URL.format(self.account) + # I think this is only used to redirect on success, which we ignore + # (it does not have to match the integration's settings in snowflake) + redirect_uri = "http://localhost:9999" + data = { + "grant_type": "refresh_token", + "refresh_token": self.token, + "redirect_uri": redirect_uri, + } + + auth = base64.b64encode( + f"{self.oauth_client_id}:{self.oauth_client_secret}".encode("ascii") + ).decode("ascii") + headers = { + "Authorization": f"Basic {auth}", + "Content-type": "application/x-www-form-urlencoded;charset=utf-8", + } + + result_json = None + max_iter = 20 + # Attempt to obtain JSON for 1 second before throwing an error + for i in range(max_iter): + result = requests.post(token_url, headers=headers, data=data) + try: + result_json = result.json() + break + except ValueError as e: + message = result.text + logger.debug( + f"Got a non-json response ({result.status_code}): \ + {e}, message: {message}" + ) + sleep(0.05) + + if result_json is None: + raise DbtDatabaseError( + f"""Did not receive valid json with access_token. + Showing json response: {result_json}""" + ) + elif "access_token" not in result_json: + raise FailedToConnectError( + "This error occurs when authentication has expired. " + "Please reauth with your auth provider." + ) + return result_json["access_token"] + + def _get_private_key(self) -> Optional[bytes]: + """Get Snowflake private key by path, from a Base64 encoded DER bytestring or None.""" + if self.private_key and self.private_key_path: + raise DbtConfigError("Cannot specify both `private_key` and `private_key_path`") + elif self.private_key: + private_key = private_key_from_string(self.private_key, self.private_key_passphrase) + elif self.private_key_path: + private_key = private_key_from_file(self.private_key_path, self.private_key_passphrase) + else: + return None + return snowflake_private_key(private_key) + + +class SnowflakeConnectionManager(SQLConnectionManager): + TYPE = "snowflake" + + @contextmanager + def exception_handler(self, sql): + try: + yield + except snowflake.connector.errors.ProgrammingError as e: + msg = str(e) + + # A class of Snowflake errors -- such as a failure from attempting to merge + # duplicate rows -- includes row values in the error message, i.e. + # [12345, "col_a_value", "col_b_value", etc...]. We don't want to log potentially + # sensitive user data. + for regex_pattern, replacement_message in ERROR_REDACTION_PATTERNS.items(): + msg = re.sub(regex_pattern, replacement_message, msg) + + logger.debug("Snowflake query id: {}".format(e.sfqid)) + logger.debug("Snowflake error: {}".format(msg)) + + if "Empty SQL statement" in msg: + logger.debug("got empty sql statement, moving on") + elif "This session does not have a current database" in msg: + raise FailedToConnectError( + ( + "{}\n\nThis error sometimes occurs when invalid " + "credentials are provided, or when your default role " + "does not have access to use the specified database. " + "Please double check your profile and try again." + ).format(msg) + ) + else: + raise DbtDatabaseError(msg) + except Exception as e: + if isinstance(e, snowflake.connector.errors.Error): + logger.debug("Snowflake query id: {}".format(e.sfqid)) + + logger.debug("Error running SQL: {}", sql) + logger.debug("Rolling back transaction.") + self.rollback_if_open() + if isinstance(e, DbtRuntimeError): + # during a sql query, an internal to dbt exception was raised. + # this sounds a lot like a signal handler and probably has + # useful information, so raise it without modification. + raise + raise DbtRuntimeError(str(e)) from e + + @classmethod + def open(cls, connection): + if connection.state == "open": + logger.debug("Connection is already open, skipping open.") + return connection + + creds = connection.credentials + timeout = creds.connect_timeout + + def connect(): + session_parameters = {} + + if creds.query_tag: + session_parameters.update({"QUERY_TAG": creds.query_tag}) + handle = None + + # In replay mode, we won't connect to a real database at all, while + # in record and diff modes we do, but insert an intermediate handle + # object which monitors native connection activity. + rec_mode = get_record_mode_from_env() + handle = None + if rec_mode != RecorderMode.REPLAY: + handle = snowflake.connector.connect( + account=creds.account, + database=creds.database, + schema=creds.schema, + warehouse=creds.warehouse, + role=creds.role, + autocommit=True, + client_session_keep_alive=creds.client_session_keep_alive, + application="dbt", + insecure_mode=creds.insecure_mode, + session_parameters=session_parameters, + **creds.auth_args(), + ) + + if rec_mode is not None: + # If using the record/replay mechanism, regardless of mode, we + # use a wrapper. + handle = SnowflakeRecordReplayHandle(handle, connection) + + return handle + + def exponential_backoff(attempt: int): + return attempt * attempt + + retryable_exceptions = [ + InternalError, + InternalServerError, + ServiceUnavailableError, + GatewayTimeoutError, + RequestTimeoutError, + BadGatewayError, + OtherHTTPRetryableError, + BindUploadError, + ] + # these two options are for backwards compatibility + if creds.retry_all: + retryable_exceptions = [Error] + elif creds.retry_on_database_errors: + retryable_exceptions.insert(0, DatabaseError) + + return cls.retry_connection( + connection, + connect=connect, + logger=logger, + retry_limit=creds.connect_retries, + retry_timeout=timeout if timeout is not None else exponential_backoff, + retryable_exceptions=retryable_exceptions, + ) + + def cancel(self, connection): + handle = connection.handle + sid = handle.session_id + + connection_name = connection.name + + sql = "select system$cancel_all_queries({})".format(sid) + + logger.debug("Cancelling query '{}' ({})".format(connection_name, sid)) + + _, cursor = self.add_query(sql) + res = cursor.fetchone() + + logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + @classmethod + def get_response(cls, cursor) -> AdapterResponse: + code = cursor.sqlstate + + if code is None: + code = "SUCCESS" + query_id = str(cursor.sfqid) if cursor.sfqid is not None else None + return AdapterResponse( + _message="{} {}".format(code, cursor.rowcount), + rows_affected=cursor.rowcount, + code=code, + query_id=query_id, + ) + + # disable transactional logic by default on Snowflake + # except for DML statements where explicitly defined + def add_begin_query(self, *args, **kwargs): + pass + + def add_commit_query(self, *args, **kwargs): + pass + + def begin(self): + pass + + def commit(self): + pass + + def clear_transaction(self): + pass + + @classmethod + def _split_queries(cls, sql): + "Splits sql statements at semicolons into discrete queries" + + sql_s = str(sql) + sql_buf = StringIO(sql_s) + split_query = snowflake.connector.util_text.split_statements(sql_buf) + return [part[0] for part in split_query] + + @staticmethod + def _fix_rows(rows: Iterable[Iterable]) -> Iterable[Iterable]: + # See note in process_results(). + for row in rows: + fixed_row = [] + for col in row: + if isinstance(col, datetime.datetime) and col.tzinfo: + offset = col.utcoffset() + assert offset is not None + offset_seconds = offset.total_seconds() + new_timezone = pytz.FixedOffset(int(offset_seconds // 60)) + col = col.astimezone(tz=new_timezone) + fixed_row.append(col) + + yield fixed_row + + @classmethod + def process_results(cls, column_names, rows): + # Override for Snowflake. The datetime objects returned by + # snowflake-connector-python are not pickleable, so we need + # to replace them with sane timezones. + return super().process_results(column_names, cls._fix_rows(rows)) + + def execute( + self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None + ) -> Tuple[AdapterResponse, "agate.Table"]: + # don't apply the query comment here + # it will be applied after ';' queries are split + from dbt_common.clients.agate_helper import empty_table + + _, cursor = self.add_query(sql, auto_begin) + response = self.get_response(cursor) + if fetch: + table = self.get_result_from_cursor(cursor, limit) + else: + table = empty_table() + return response, table + + def add_standard_query(self, sql: str, **kwargs) -> Tuple[Connection, Any]: + # This is the happy path for a single query. Snowflake has a few odd behaviors that + # require preprocessing within the 'add_query' method below. + return super().add_query(self._add_query_comment(sql), **kwargs) + + def add_query( + self, + sql: str, + auto_begin: bool = True, + bindings: Optional[Any] = None, + abridge_sql_log: bool = False, + ) -> Tuple[Connection, Any]: + if bindings: + # The snowflake connector is stricter than, e.g., psycopg2 - + # which allows any iterable thing to be passed as a binding. + bindings = tuple(bindings) + + stripped_queries = self._stripped_queries(sql) + + if set(query.lower() for query in stripped_queries).issubset({"begin;", "commit;"}): + connection, cursor = self._add_begin_commit_only_queries( + stripped_queries, + auto_begin=auto_begin, + bindings=bindings, + abridge_sql_log=abridge_sql_log, + ) + else: + connection, cursor = self._add_standard_queries( + stripped_queries, + auto_begin=auto_begin, + bindings=bindings, + abridge_sql_log=abridge_sql_log, + ) + + if cursor is None: + self._raise_cursor_not_found_error(sql) + + return connection, cursor + + def _stripped_queries(self, sql: str) -> List[str]: + def strip_query(query): + """ + hack -- after the last ';', remove comments and don't run + empty queries. this avoids using exceptions as flow control, + and also allows us to return the status of the last cursor + """ + without_comments_re = re.compile( + r"(\".*?\"|\'.*?\')|(/\*.*?\*/|--[^\r\n]*$)", re.MULTILINE + ) + return re.sub(without_comments_re, "", query).strip() + + return [query for query in self._split_queries(sql) if strip_query(query) != ""] + + def _add_begin_commit_only_queries( + self, queries: List[str], **kwargs + ) -> Tuple[Connection, Any]: + # if all we get is `begin;` and/or `commit;` + # raise a warning, then run as standard queries to avoid an error downstream + message = ( + "Explicit transactional logic should be used only to wrap " + "DML logic (MERGE, DELETE, UPDATE, etc). The keywords BEGIN; and COMMIT; should " + "be placed directly before and after your DML statement, rather than in separate " + "statement calls or run_query() macros." + ) + logger.warning(line_wrap_message(warning_tag(message))) + + for query in queries: + connection, cursor = self.add_standard_query(query, **kwargs) + return connection, cursor + + def _add_standard_queries(self, queries: List[str], **kwargs) -> Tuple[Connection, Any]: + for query in queries: + # Even though we turn off transactions by default for Snowflake, + # the user/macro has passed them *explicitly*, probably to wrap a DML statement + # This also has the effect of ignoring "commit" in the RunResult for this model + # https://github.com/dbt-labs/dbt-snowflake/issues/147 + if query.lower() == "begin;": + super().add_begin_query() + elif query.lower() == "commit;": + super().add_commit_query() + else: + # This adds a query comment to *every* statement + # https://github.com/dbt-labs/dbt-snowflake/issues/140 + connection, cursor = self.add_standard_query(query, **kwargs) + return connection, cursor + + def _raise_cursor_not_found_error(self, sql: str): + conn = self.get_thread_connection() + try: + conn_name = conn.name + except AttributeError: + conn_name = None + + raise DbtRuntimeError( + f"""Tried to run an empty query on model '{conn_name or ""}'. If you are """ + f"""conditionally running\nsql, e.g. in a model hook, make """ + f"""sure your `else` clause contains valid sql!\n\n""" + f"""Provided SQL:\n{sql}""" + ) + + def release(self): + """Reuse connections by deferring release until adapter context manager in core + resets adapters. This cleanup_all happens before Python teardown. Idle connections + incur no costs while waiting in the connection pool.""" + if self.profile.credentials.reuse_connections: + return + super().release() + + @classmethod + def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: + assert isinstance(type_code, int) + return snowflake.connector.constants.FIELD_ID_TO_NAME[type_code] diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/constants.py b/dbt-snowflake/src/dbt/adapters/snowflake/constants.py new file mode 100644 index 000000000..9c475dcd9 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/constants.py @@ -0,0 +1 @@ +DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS = "3.9" diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/impl.py b/dbt-snowflake/src/dbt/adapters/snowflake/impl.py new file mode 100644 index 000000000..6ae8ef183 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/impl.py @@ -0,0 +1,451 @@ +from dataclasses import dataclass +from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple, TYPE_CHECKING + +from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport +from dbt.adapters.base.meta import available +from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability +from dbt.adapters.contracts.relation import RelationConfig +from dbt.adapters.sql import SQLAdapter +from dbt.adapters.sql.impl import ( + LIST_SCHEMAS_MACRO_NAME, + LIST_RELATIONS_MACRO_NAME, +) +from dbt_common.behavior_flags import BehaviorFlag +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.contracts.metadata import ( + TableMetadata, + StatsDict, + StatsItem, + CatalogTable, + ColumnMetadata, +) +from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError +from dbt_common.utils import filter_null_values + +from dbt.adapters.snowflake import constants +from dbt.adapters.snowflake.relation_configs import ( + SnowflakeRelationType, + TableFormat, +) + +from dbt.adapters.snowflake import SnowflakeColumn +from dbt.adapters.snowflake import SnowflakeConnectionManager +from dbt.adapters.snowflake import SnowflakeRelation + +if TYPE_CHECKING: + import agate + +SHOW_OBJECT_METADATA_MACRO_NAME = "snowflake__show_object_metadata" + + +@dataclass +class SnowflakeConfig(AdapterConfig): + transient: Optional[bool] = None + cluster_by: Optional[Union[str, List[str]]] = None + automatic_clustering: Optional[bool] = None + secure: Optional[bool] = None + copy_grants: Optional[bool] = None + snowflake_warehouse: Optional[str] = None + query_tag: Optional[str] = None + tmp_relation_type: Optional[str] = None + merge_update_columns: Optional[str] = None + target_lag: Optional[str] = None + + # extended formats + table_format: Optional[str] = None + external_volume: Optional[str] = None + base_location_root: Optional[str] = None + base_location_subpath: Optional[str] = None + + +class SnowflakeAdapter(SQLAdapter): + Relation = SnowflakeRelation + Column = SnowflakeColumn + ConnectionManager = SnowflakeConnectionManager + + AdapterSpecificConfigs = SnowflakeConfig + + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.NOT_ENFORCED, + ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, + } + + _capabilities: CapabilityDict = CapabilityDict( + { + Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), + Capability.GetCatalogForSingleRelation: CapabilitySupport(support=Support.Full), + Capability.MicrobatchConcurrency: CapabilitySupport(support=Support.Full), + } + ) + + @property + def _behavior_flags(self) -> List[BehaviorFlag]: + return [ + { + "name": "enable_iceberg_materializations", + "default": False, + "description": ( + "Enabling Iceberg materializations introduces latency to metadata queries, " + "specifically within the list_relations_without_caching macro. Since Iceberg " + "benefits only those actively using it, we've made this behavior opt-in to " + "prevent unnecessary latency for other users." + ), + "docs_url": "https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format", + } + ] + + @classmethod + def date_function(cls): + return "CURRENT_TIMESTAMP()" + + @classmethod + def _catalog_filter_table( + cls, table: "agate.Table", used_schemas: FrozenSet[Tuple[str, str]] + ) -> "agate.Table": + # On snowflake, users can set QUOTED_IDENTIFIERS_IGNORE_CASE, so force + # the column names to their lowercased forms. + lowered = table.rename(column_names=[c.lower() for c in table.column_names]) + return super()._catalog_filter_table(lowered, used_schemas) + + def _make_match_kwargs(self, database, schema, identifier): + quoting = self.config.quoting + if identifier is not None and quoting["identifier"] is False: + identifier = identifier.upper() + + if schema is not None and quoting["schema"] is False: + schema = schema.upper() + + if database is not None and quoting["database"] is False: + database = database.upper() + + return filter_null_values( + {"identifier": identifier, "schema": schema, "database": database} + ) + + def _get_warehouse(self) -> str: + _, table = self.execute("select current_warehouse() as warehouse", fetch=True) + if len(table) == 0 or len(table[0]) == 0: + # can this happen? + raise DbtRuntimeError("Could not get current warehouse: no results") + return str(table[0][0]) + + def _use_warehouse(self, warehouse: str): + """Use the given warehouse. Quotes are never applied.""" + self.execute("use warehouse {}".format(warehouse)) + + def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]: + default_warehouse = self.config.credentials.warehouse + warehouse = config.get("snowflake_warehouse", default_warehouse) + if warehouse == default_warehouse or warehouse is None: + return None + previous = self._get_warehouse() + self._use_warehouse(warehouse) + return previous + + def post_model_hook(self, config: Mapping[str, Any], context: Optional[str]) -> None: + if context is not None: + self._use_warehouse(context) + + def list_schemas(self, database: str) -> List[str]: + try: + results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) + except DbtDatabaseError as exc: + msg = f"Database error while listing schemas in database " f'"{database}"\n{exc}' + raise DbtRuntimeError(msg) + # this uses 'show terse schemas in database', and the column name we + # want is 'name' + + return [row["name"] for row in results] + + def get_columns_in_relation(self, relation): + try: + return super().get_columns_in_relation(relation) + except DbtDatabaseError as exc: + if "does not exist or not authorized" in str(exc): + return [] + else: + raise + + def _show_object_metadata(self, relation: SnowflakeRelation) -> Optional[dict]: + try: + kwargs = {"relation": relation} + results = self.execute_macro(SHOW_OBJECT_METADATA_MACRO_NAME, kwargs=kwargs) + + if len(results) == 0: + return None + + return results + except DbtDatabaseError: + return None + + def get_catalog_for_single_relation( + self, relation: SnowflakeRelation + ) -> Optional[CatalogTable]: + object_metadata = self._show_object_metadata(relation.as_case_sensitive()) + + if not object_metadata: + return None + + row = object_metadata[0] + + is_dynamic = row.get("is_dynamic") in ("Y", "YES") + kind = row.get("kind") + + if is_dynamic and kind == str(SnowflakeRelationType.Table).upper(): + table_type = str(SnowflakeRelationType.DynamicTable).upper() + else: + table_type = kind + + # https://docs.snowflake.com/en/sql-reference/sql/show-views#output + # Note: we don't support materialized views in dbt-snowflake + is_view = kind == str(SnowflakeRelationType.View).upper() + + table_metadata = TableMetadata( + type=table_type, + schema=row.get("schema_name"), + name=row.get("name"), + database=row.get("database_name"), + comment=row.get("comment"), + owner=row.get("owner"), + ) + + stats_dict: StatsDict = { + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=row.get("rows"), + include=(not is_view), + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=row.get("bytes"), + include=(not is_view), + description="Size of the table as reported by Snowflake", + ), + } + + catalog_columns = { + c.column: ColumnMetadata(type=c.dtype, index=i + 1, name=c.column) + for i, c in enumerate(self.get_columns_in_relation(relation)) + } + + return CatalogTable( + metadata=table_metadata, + columns=catalog_columns, + stats=stats_dict, + ) + + def list_relations_without_caching( + self, schema_relation: SnowflakeRelation + ) -> List[SnowflakeRelation]: + kwargs = {"schema_relation": schema_relation} + + try: + schema_objects = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + except DbtDatabaseError as exc: + # if the schema doesn't exist, we just want to return. + # Alternatively, we could query the list of schemas before we start + # and skip listing the missing ones, which sounds expensive. + # "002043 (02000)" is error code for "object does not exist or is not found" + # The error message text may vary across languages, but the error code is expected to be more stable + if "002043 (02000)" in str(exc): + return [] + raise + + # this can be collapsed once Snowflake adds is_iceberg to show objects + columns = ["database_name", "schema_name", "name", "kind", "is_dynamic"] + if self.behavior.enable_iceberg_materializations.no_warn: + # The QUOTED_IDENTIFIERS_IGNORE_CASE setting impacts column names like + # is_iceberg which is created by dbt, but it does not affect the case + # of column values in Snowflake's SHOW OBJECTS query! This + # normalization step ensures metadata queries are handled consistently. + schema_objects = schema_objects.rename(column_names={"IS_ICEBERG": "is_iceberg"}) + columns.append("is_iceberg") + + return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] + + def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation: + # this can be collapsed once Snowflake adds is_iceberg to show objects + if self.behavior.enable_iceberg_materializations.no_warn: + database, schema, identifier, relation_type, is_dynamic, is_iceberg = result + else: + database, schema, identifier, relation_type, is_dynamic = result + is_iceberg = "N" + + try: + relation_type = self.Relation.get_relation_type(relation_type.lower()) + except ValueError: + relation_type = self.Relation.External + + if relation_type == self.Relation.Table and is_dynamic == "Y": + relation_type = self.Relation.DynamicTable + + table_format = TableFormat.ICEBERG if is_iceberg in ("Y", "YES") else TableFormat.DEFAULT + + quote_policy = {"database": True, "schema": True, "identifier": True} + + return self.Relation.create( + database=database, + schema=schema, + identifier=identifier, + type=relation_type, + table_format=table_format, + quote_policy=quote_policy, + ) + + def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str: + quote_columns: bool = False + if isinstance(quote_config, bool): + quote_columns = quote_config + elif quote_config is None: + pass + else: + msg = ( + f'The seed configuration value of "quote_columns" has an ' + f"invalid type {type(quote_config)}" + ) + raise CompilationError(msg) + + if quote_columns: + return self.quote(column) + else: + return column + + @available + def standardize_grants_dict(self, grants_table: "agate.Table") -> dict: + grants_dict: Dict[str, Any] = {} + + for row in grants_table: + grantee = row["grantee_name"] + granted_to = row["granted_to"] + privilege = row["privilege"] + if privilege != "OWNERSHIP" and granted_to not in ["SHARE", "DATABASE_ROLE"]: + if privilege in grants_dict.keys(): + grants_dict[privilege].append(grantee) + else: + grants_dict.update({privilege: [grantee]}) + return grants_dict + + def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: + return f"DATEADD({interval}, {number}, {add_to})" + + def submit_python_job(self, parsed_model: dict, compiled_code: str): + schema = parsed_model["schema"] + database = parsed_model["database"] + identifier = parsed_model["alias"] + python_version = parsed_model["config"].get( + "python_version", constants.DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS + ) + + packages = parsed_model["config"].get("packages", []) + imports = parsed_model["config"].get("imports", []) + external_access_integrations = parsed_model["config"].get( + "external_access_integrations", [] + ) + secrets = parsed_model["config"].get("secrets", {}) + # adding default packages we need to make python model work + default_packages = ["snowflake-snowpark-python"] + package_names = [package.split("==")[0] for package in packages] + for default_package in default_packages: + if default_package not in package_names: + packages.append(default_package) + packages = "', '".join(packages) + imports = "', '".join(imports) + external_access_integrations = ", ".join(external_access_integrations) + secrets = ", ".join(f"'{key}' = {value}" for key, value in secrets.items()) + + # we can't pass empty imports, external_access_integrations or secrets clause to snowflake + if imports: + imports = f"IMPORTS = ('{imports}')" + if external_access_integrations: + # Black is trying to make this a tuple. + # fmt: off + external_access_integrations = f"EXTERNAL_ACCESS_INTEGRATIONS = ({external_access_integrations})" + if secrets: + secrets = f"SECRETS = ({secrets})" + + if self.config.args.SEND_ANONYMOUS_USAGE_STATS: + snowpark_telemetry_string = "dbtLabs_dbtPython" + snowpark_telemetry_snippet = f""" +import sys +sys._xoptions['snowflake_partner_attribution'].append("{snowpark_telemetry_string}")""" + else: + snowpark_telemetry_snippet = "" + + common_procedure_code = f""" +RETURNS STRING +LANGUAGE PYTHON +RUNTIME_VERSION = '{python_version}' +PACKAGES = ('{packages}') +{external_access_integrations} +{secrets} +{imports} +HANDLER = 'main' +EXECUTE AS CALLER +AS +$$ +{snowpark_telemetry_snippet} + +{compiled_code} +$$""" + + use_anonymous_sproc = parsed_model["config"].get("use_anonymous_sproc", True) + if use_anonymous_sproc: + proc_name = f"{identifier}__dbt_sp" + python_stored_procedure = f""" +WITH {proc_name} AS PROCEDURE () +{common_procedure_code} +CALL {proc_name}(); + """ + else: + proc_name = f"{database}.{schema}.{identifier}__dbt_sp" + python_stored_procedure = f""" +CREATE OR REPLACE PROCEDURE {proc_name} () +{common_procedure_code}; +CALL {proc_name}(); + + """ + response, _ = self.execute(python_stored_procedure, auto_begin=False, fetch=False) + if not use_anonymous_sproc: + self.execute( + f"drop procedure if exists {proc_name}()", + auto_begin=False, + fetch=False, + ) + return response + + def valid_incremental_strategies(self): + return ["append", "merge", "delete+insert", "microbatch"] + + def debug_query(self): + """Override for DebugTask method""" + self.execute("select 1 as id") + + @classmethod + def _get_adapter_specific_run_info(cls, config: RelationConfig) -> Dict[str, Any]: + table_format: Optional[str] = None + if ( + config + and hasattr(config, "_extra") + and (relation_format := config._extra.get("table_format")) + ): + table_format = relation_format + + return { + "adapter_type": "snowflake", + "table_format": table_format, + } diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/record/__init__.py b/dbt-snowflake/src/dbt/adapters/snowflake/record/__init__.py new file mode 100644 index 000000000..f763dc3a4 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/record/__init__.py @@ -0,0 +1,2 @@ +from dbt.adapters.snowflake.record.cursor.cursor import SnowflakeRecordReplayCursor +from dbt.adapters.snowflake.record.handle import SnowflakeRecordReplayHandle diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/cursor.py b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/cursor.py new file mode 100644 index 000000000..a07468867 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/cursor.py @@ -0,0 +1,21 @@ +from dbt_common.record import record_function + +from dbt.adapters.record import RecordReplayCursor +from dbt.adapters.snowflake.record.cursor.sfqid import CursorGetSfqidRecord +from dbt.adapters.snowflake.record.cursor.sqlstate import CursorGetSqlStateRecord + + +class SnowflakeRecordReplayCursor(RecordReplayCursor): + """A custom extension of RecordReplayCursor that adds the sqlstate + and sfqid properties which are specific to snowflake-connector.""" + + @property + @property + @record_function(CursorGetSqlStateRecord, method=True, id_field_name="connection_name") + def sqlstate(self): + return self.native_cursor.sqlstate + + @property + @record_function(CursorGetSfqidRecord, method=True, id_field_name="connection_name") + def sfqid(self): + return self.native_cursor.sfqid diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sfqid.py b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sfqid.py new file mode 100644 index 000000000..e39c857d3 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sfqid.py @@ -0,0 +1,21 @@ +import dataclasses +from typing import Optional + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetSfqidParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetSfqidResult: + msg: Optional[str] + + +@Recorder.register_record_type +class CursorGetSfqidRecord(Record): + params_cls = CursorGetSfqidParams + result_cls = CursorGetSfqidResult + group = "Database" diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sqlstate.py b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sqlstate.py new file mode 100644 index 000000000..5619058fd --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/record/cursor/sqlstate.py @@ -0,0 +1,21 @@ +import dataclasses +from typing import Optional + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetSqlStateParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetSqlStateResult: + msg: Optional[str] + + +@Recorder.register_record_type +class CursorGetSqlStateRecord(Record): + params_cls = CursorGetSqlStateParams + result_cls = CursorGetSqlStateResult + group = "Database" diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/record/handle.py b/dbt-snowflake/src/dbt/adapters/snowflake/record/handle.py new file mode 100644 index 000000000..046bb911b --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/record/handle.py @@ -0,0 +1,12 @@ +from dbt.adapters.record import RecordReplayHandle + +from dbt.adapters.snowflake.record.cursor.cursor import SnowflakeRecordReplayCursor + + +class SnowflakeRecordReplayHandle(RecordReplayHandle): + """A custom extension of RecordReplayHandle that returns a + snowflake-connector-specific SnowflakeRecordReplayCursor object.""" + + def cursor(self): + cursor = None if self.native_handle is None else self.native_handle.cursor() + return SnowflakeRecordReplayCursor(cursor, self.connection) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation.py new file mode 100644 index 000000000..f3ee3e510 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation.py @@ -0,0 +1,274 @@ +import textwrap + +from dataclasses import dataclass, field +from typing import FrozenSet, Optional, Type, Iterator, Tuple + + +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.contracts.relation import ComponentName, RelationConfig +from dbt.adapters.events.types import AdapterEventWarning, AdapterEventDebug +from dbt.adapters.relation_configs import ( + RelationConfigBase, + RelationConfigChangeAction, + RelationResults, +) +from dbt.adapters.utils import classproperty +from dbt_common.exceptions import DbtRuntimeError +from dbt_common.events.functions import fire_event, warn_or_error + +from dbt.adapters.snowflake.relation_configs import ( + RefreshMode, + SnowflakeCatalogConfigChange, + SnowflakeDynamicTableConfig, + SnowflakeDynamicTableConfigChangeset, + SnowflakeDynamicTableRefreshModeConfigChange, + SnowflakeDynamicTableTargetLagConfigChange, + SnowflakeDynamicTableWarehouseConfigChange, + TableFormat, + SnowflakeQuotePolicy, + SnowflakeRelationType, +) + + +@dataclass(frozen=True, eq=False, repr=False) +class SnowflakeRelation(BaseRelation): + type: Optional[SnowflakeRelationType] = None + table_format: str = TableFormat.DEFAULT + quote_policy: SnowflakeQuotePolicy = field(default_factory=lambda: SnowflakeQuotePolicy()) + require_alias: bool = False + relation_configs = { + SnowflakeRelationType.DynamicTable: SnowflakeDynamicTableConfig, + } + renameable_relations: FrozenSet[SnowflakeRelationType] = field( + default_factory=lambda: frozenset( + { + SnowflakeRelationType.Table, # type: ignore + SnowflakeRelationType.View, # type: ignore + } + ) + ) + + replaceable_relations: FrozenSet[SnowflakeRelationType] = field( + default_factory=lambda: frozenset( + { + SnowflakeRelationType.DynamicTable, # type: ignore + SnowflakeRelationType.Table, # type: ignore + SnowflakeRelationType.View, # type: ignore + } + ) + ) + + @property + def is_dynamic_table(self) -> bool: + return self.type == SnowflakeRelationType.DynamicTable + + @property + def is_iceberg_format(self) -> bool: + return self.table_format == TableFormat.ICEBERG + + @classproperty + def DynamicTable(cls) -> str: + return str(SnowflakeRelationType.DynamicTable) + + @classproperty + def get_relation_type(cls) -> Type[SnowflakeRelationType]: + return SnowflakeRelationType + + @classmethod + def from_config(cls, config: RelationConfig) -> RelationConfigBase: + relation_type: str = config.config.materialized + + if relation_config := cls.relation_configs.get(relation_type): + return relation_config.from_relation_config(config) + + raise DbtRuntimeError( + f"from_config() is not supported for the provided relation type: {relation_type}" + ) + + @classmethod + def dynamic_table_config_changeset( + cls, relation_results: RelationResults, relation_config: RelationConfig + ) -> Optional[SnowflakeDynamicTableConfigChangeset]: + existing_dynamic_table = SnowflakeDynamicTableConfig.from_relation_results( + relation_results + ) + new_dynamic_table = SnowflakeDynamicTableConfig.from_relation_config(relation_config) + + config_change_collection = SnowflakeDynamicTableConfigChangeset() + + if new_dynamic_table.target_lag != existing_dynamic_table.target_lag: + config_change_collection.target_lag = SnowflakeDynamicTableTargetLagConfigChange( + action=RelationConfigChangeAction.alter, + context=new_dynamic_table.target_lag, + ) + + if new_dynamic_table.snowflake_warehouse != existing_dynamic_table.snowflake_warehouse: + config_change_collection.snowflake_warehouse = ( + SnowflakeDynamicTableWarehouseConfigChange( + action=RelationConfigChangeAction.alter, + context=new_dynamic_table.snowflake_warehouse, + ) + ) + + if ( + new_dynamic_table.refresh_mode != RefreshMode.AUTO + and new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode + ): + config_change_collection.refresh_mode = SnowflakeDynamicTableRefreshModeConfigChange( + action=RelationConfigChangeAction.create, + context=new_dynamic_table.refresh_mode, + ) + + if new_dynamic_table.catalog != existing_dynamic_table.catalog: + config_change_collection.catalog = SnowflakeCatalogConfigChange( + action=RelationConfigChangeAction.create, + context=new_dynamic_table.catalog, + ) + + if config_change_collection.has_changes: + return config_change_collection + return None + + def as_case_sensitive(self) -> "SnowflakeRelation": + path_part_map = {} + + for path in ComponentName: + if self.include_policy.get_part(path): + part = self.path.get_part(path) + if part: + if self.quote_policy.get_part(path): + path_part_map[path] = part + else: + path_part_map[path] = part.upper() + + return self.replace_path(**path_part_map) + + @property + def can_be_renamed(self) -> bool: + """ + Standard tables and dynamic tables can be renamed, but Snowflake does not support renaming iceberg relations. + The iceberg standard does support renaming, so this may change in the future. + """ + return self.type in self.renameable_relations and not self.is_iceberg_format + + def get_ddl_prefix_for_create(self, config: RelationConfig, temporary: bool) -> str: + """ + This macro renders the appropriate DDL prefix during the create_table_as + macro. It decides based on mutually exclusive table configuration options: + + - TEMPORARY: Indicates a table that exists only for the duration of the session. + - ICEBERG: A specific storage format that requires a distinct DDL layout. + - TRANSIENT: A table similar to a permanent table but without fail-safe. + + Additional Caveats for Iceberg models: + - transient=true throws a warning because Iceberg does not support transient tables + - A temporary relation is never an Iceberg relation because Iceberg does not + support temporary relations. + """ + + transient_explicitly_set_true: bool = config.get("transient", False) + + # Temporary tables are a Snowflake feature that do not exist in the + # Iceberg framework. We ignore the Iceberg status of the model. + if temporary: + return "temporary" + elif self.is_iceberg_format: + # Log a warning that transient=true on an Iceberg relation is ignored. + if transient_explicitly_set_true: + warn_or_error( + AdapterEventWarning( + base_msg=( + "Iceberg format relations cannot be transient. Please " + "remove either the transient or iceberg config options " + f"from {self.path.database}.{self.path.schema}." + f"{self.path.identifier}. If left unmodified, dbt will " + "ignore 'transient'." + ) + ) + ) + + return "iceberg" + + # Always supply transient on table create DDL unless user specifically sets + # transient to false or unset. Might as well update the object attribute too! + elif transient_explicitly_set_true or config.get("transient", True): + return "transient" + else: + return "" + + def get_ddl_prefix_for_alter(self) -> str: + """All ALTER statements on Iceberg tables require an ICEBERG prefix""" + if self.is_iceberg_format: + return "iceberg" + else: + return "" + + def get_iceberg_ddl_options(self, config: RelationConfig) -> str: + # If the base_location_root config is supplied, overwrite the default value ("_dbt/") + base_location: str = ( + f"{config.get('base_location_root', '_dbt')}/{self.schema}/{self.name}" + ) + + if subpath := config.get("base_location_subpath"): + base_location += f"/{subpath}" + + iceberg_ddl_predicates: str = f""" + external_volume = '{config.get('external_volume')}' + catalog = 'snowflake' + base_location = '{base_location}' + """ + return textwrap.indent(textwrap.dedent(iceberg_ddl_predicates), " " * 10) + + def __drop_conditions(self, old_relation: "SnowflakeRelation") -> Iterator[Tuple[bool, str]]: + drop_view_message: str = ( + f"Dropping relation {old_relation} because it is a view and target relation {self} " + f"is of type {self.type}." + ) + + drop_table_for_iceberg_message: str = ( + f"Dropping relation {old_relation} because it is a default format table " + f"and target relation {self} is an Iceberg format table." + ) + + drop_iceberg_for_table_message: str = ( + f"Dropping relation {old_relation} because it is an Iceberg format table " + f"and target relation {self} is a default format table." + ) + + # An existing view must be dropped for model to build into a table". + yield (not old_relation.is_table, drop_view_message) + # An existing table must be dropped for model to build into an Iceberg table. + yield ( + old_relation.is_table + and not old_relation.is_iceberg_format + and self.is_iceberg_format, + drop_table_for_iceberg_message, + ) + # existing Iceberg table must be dropped for model to build into a table. + yield ( + old_relation.is_table + and old_relation.is_iceberg_format + and not self.is_iceberg_format, + drop_iceberg_for_table_message, + ) + + def needs_to_drop(self, old_relation: Optional["SnowflakeRelation"]) -> bool: + """ + To convert between Iceberg and non-Iceberg relations, a preemptive drop is + required. + + drops cause latency, but it should be a relatively infrequent occurrence. + + Some Boolean expression below are logically redundant, but this is done for easier + readability. + """ + + if old_relation is None: + return False + + for condition, message in self.__drop_conditions(old_relation): + if condition: + fire_event(AdapterEventDebug(base_msg=message)) + return True + + return False diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/__init__.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/__init__.py new file mode 100644 index 000000000..67f7644d2 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/__init__.py @@ -0,0 +1,18 @@ +from dbt.adapters.snowflake.relation_configs.catalog import ( + SnowflakeCatalogConfig, + SnowflakeCatalogConfigChange, +) +from dbt.adapters.snowflake.relation_configs.dynamic_table import ( + RefreshMode, + SnowflakeDynamicTableConfig, + SnowflakeDynamicTableConfigChangeset, + SnowflakeDynamicTableRefreshModeConfigChange, + SnowflakeDynamicTableWarehouseConfigChange, + SnowflakeDynamicTableTargetLagConfigChange, +) +from dbt.adapters.snowflake.relation_configs.formats import TableFormat +from dbt.adapters.snowflake.relation_configs.policies import ( + SnowflakeIncludePolicy, + SnowflakeQuotePolicy, + SnowflakeRelationType, +) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/base.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/base.py new file mode 100644 index 000000000..a1ef79684 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/base.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, TYPE_CHECKING +from dbt.adapters.base.relation import Policy +from dbt.adapters.relation_configs import ( + RelationConfigBase, + RelationResults, +) + +from dbt.adapters.contracts.relation import ComponentName, RelationConfig + +from dbt.adapters.snowflake.relation_configs.policies import ( + SnowflakeIncludePolicy, + SnowflakeQuotePolicy, +) + +if TYPE_CHECKING: + # Imported downfile for specific row gathering function. + import agate + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeRelationConfigBase(RelationConfigBase): + """ + This base class implements a few boilerplate methods and provides some light structure for Snowflake relations. + """ + + @classmethod + def include_policy(cls) -> Policy: + return SnowflakeIncludePolicy() + + @classmethod + def quote_policy(cls) -> Policy: + return SnowflakeQuotePolicy() + + @classmethod + def from_relation_config(cls, relation_config: RelationConfig): + relation_config_dict = cls.parse_relation_config(relation_config) + relation = cls.from_dict(relation_config_dict) + return relation + + @classmethod + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict: + raise NotImplementedError( + "`parse_relation_config()` needs to be implemented on this RelationConfigBase instance" + ) + + @classmethod + def from_relation_results(cls, relation_results: RelationResults): + relation_config = cls.parse_relation_results(relation_results) + relation = cls.from_dict(relation_config) + return relation + + @classmethod + def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: + raise NotImplementedError( + "`parse_relation_results()` needs to be implemented on this RelationConfigBase instance" + ) + + @classmethod + def _render_part(cls, component: ComponentName, value: Optional[str]) -> Optional[str]: + if cls.include_policy().get_part(component) and value: + if cls.quote_policy().get_part(component): + return f'"{value}"' + return value.lower() + return None + + @classmethod + def _get_first_row(cls, results: "agate.Table") -> "agate.Row": + try: + return results.rows[0] + except IndexError: + import agate + + return agate.Row(values=set()) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/catalog.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/catalog.py new file mode 100644 index 000000000..c8d7de40f --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/catalog.py @@ -0,0 +1,125 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, TYPE_CHECKING, Set, List + +if TYPE_CHECKING: + import agate + +from dbt.adapters.relation_configs import ( + RelationConfigChange, + RelationResults, + RelationConfigValidationMixin, + RelationConfigValidationRule, +) +from dbt.adapters.contracts.relation import RelationConfig +from dbt_common.exceptions import DbtConfigError +from typing_extensions import Self + +from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase +from dbt.adapters.snowflake.relation_configs.formats import TableFormat + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeCatalogConfig(SnowflakeRelationConfigBase, RelationConfigValidationMixin): + """ + This config follow the specs found here: + https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table + https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table + + The following parameters are configurable by dbt: + - table_format: format for interfacing with the table, e.g. default, iceberg + - external_volume: name of the external volume in Snowflake + - base_location: the directory within the external volume that contains the data + *Note*: This directory can’t be changed after you create a table. + + The following parameters are not currently configurable by dbt: + - name: snowflake + """ + + table_format: Optional[TableFormat] = TableFormat.default() + name: Optional[str] = "SNOWFLAKE" + external_volume: Optional[str] = None + base_location: Optional[str] = None + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + return { + RelationConfigValidationRule( + (self.table_format == "default") + or (self.table_format == "iceberg" and self.base_location is not None), + DbtConfigError("Please provide a `base_location` when using iceberg"), + ), + RelationConfigValidationRule( + (self.table_format == "default") + or (self.table_format == "iceberg" and self.name == "SNOWFLAKE"), + DbtConfigError( + "Only Snowflake catalogs are currently supported when using iceberg" + ), + ), + } + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any]) -> Self: + kwargs_dict = { + "name": config_dict.get("name"), + "external_volume": config_dict.get("external_volume"), + "base_location": config_dict.get("base_location"), + } + if table_format := config_dict.get("table_format"): + kwargs_dict["table_format"] = TableFormat(table_format) + return super().from_dict(kwargs_dict) + + @classmethod + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: + + if relation_config.config.extra.get("table_format") is None: + return {} + + config_dict = { + "table_format": relation_config.config.extra.get("table_format"), + "name": "SNOWFLAKE", # this is not currently configurable + } + + if external_volume := relation_config.config.extra.get("external_volume"): + config_dict["external_volume"] = external_volume + + catalog_dirs: List[str] = ["_dbt", relation_config.schema, relation_config.name] + if base_location_subpath := relation_config.config.extra.get("base_location_subpath"): + catalog_dirs.append(base_location_subpath) + config_dict["base_location"] = "/".join(catalog_dirs) + + return config_dict + + @classmethod + def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: + # this try block can be removed once enable_iceberg_materializations is retired + try: + catalog_results: "agate.Table" = relation_results["catalog"] + except KeyError: + # this happens when `enable_iceberg_materializations` is turned off + return {} + + if len(catalog_results) == 0: + # this happens when the dynamic table is a standard dynamic table (e.g. not iceberg) + return {} + + # for now, if we get catalog results, it's because this is an iceberg table + # this is because we only run `show iceberg tables` to get catalog metadata + # this will need to be updated once this is in `show objects` + catalog: "agate.Row" = catalog_results.rows[0] + config_dict = { + "table_format": "iceberg", + "name": catalog.get("catalog_name"), + "external_volume": catalog.get("external_volume_name"), + "base_location": catalog.get("base_location"), + } + + return config_dict + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeCatalogConfigChange(RelationConfigChange): + context: Optional[SnowflakeCatalogConfig] = None + + @property + def requires_full_refresh(self) -> bool: + return True diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/dynamic_table.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/dynamic_table.py new file mode 100644 index 000000000..7361df80a --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/dynamic_table.py @@ -0,0 +1,175 @@ +from dataclasses import dataclass +from typing import Optional, Dict, Any, TYPE_CHECKING + +from dbt.adapters.relation_configs import RelationConfigChange, RelationResults +from dbt.adapters.contracts.relation import RelationConfig +from dbt.adapters.contracts.relation import ComponentName +from dbt_common.dataclass_schema import StrEnum # doesn't exist in standard library until py3.11 +from typing_extensions import Self + +from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase +from dbt.adapters.snowflake.relation_configs.catalog import ( + SnowflakeCatalogConfig, + SnowflakeCatalogConfigChange, +) + + +if TYPE_CHECKING: + import agate + + +class RefreshMode(StrEnum): + AUTO = "AUTO" + FULL = "FULL" + INCREMENTAL = "INCREMENTAL" + + @classmethod + def default(cls) -> Self: + return cls("AUTO") + + +class Initialize(StrEnum): + ON_CREATE = "ON_CREATE" + ON_SCHEDULE = "ON_SCHEDULE" + + @classmethod + def default(cls) -> Self: + return cls("ON_CREATE") + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeDynamicTableConfig(SnowflakeRelationConfigBase): + """ + This config follow the specs found here: + https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table + + The following parameters are configurable by dbt: + - name: name of the dynamic table + - query: the query behind the table + - target_lag: the maximum amount of time that the dynamic table’s content should lag behind updates to the base tables + - snowflake_warehouse: the name of the warehouse that provides the compute resources for refreshing the dynamic table + - refresh_mode: specifies the refresh type for the dynamic table + - initialize: specifies the behavior of the initial refresh of the dynamic table + + There are currently no non-configurable parameters. + """ + + name: str + schema_name: str + database_name: str + query: str + target_lag: str + snowflake_warehouse: str + catalog: SnowflakeCatalogConfig + refresh_mode: Optional[RefreshMode] = RefreshMode.default() + initialize: Optional[Initialize] = Initialize.default() + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any]) -> Self: + kwargs_dict = { + "name": cls._render_part(ComponentName.Identifier, config_dict.get("name")), + "schema_name": cls._render_part(ComponentName.Schema, config_dict.get("schema_name")), + "database_name": cls._render_part( + ComponentName.Database, config_dict.get("database_name") + ), + "query": config_dict.get("query"), + "target_lag": config_dict.get("target_lag"), + "snowflake_warehouse": config_dict.get("snowflake_warehouse"), + "catalog": SnowflakeCatalogConfig.from_dict(config_dict["catalog"]), + "refresh_mode": config_dict.get("refresh_mode"), + "initialize": config_dict.get("initialize"), + } + + return super().from_dict(kwargs_dict) + + @classmethod + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: + config_dict = { + "name": relation_config.identifier, + "schema_name": relation_config.schema, + "database_name": relation_config.database, + "query": relation_config.compiled_code, + "target_lag": relation_config.config.extra.get("target_lag"), + "snowflake_warehouse": relation_config.config.extra.get("snowflake_warehouse"), + "catalog": SnowflakeCatalogConfig.parse_relation_config(relation_config), + } + + if refresh_mode := relation_config.config.extra.get("refresh_mode"): + config_dict["refresh_mode"] = refresh_mode.upper() + + if initialize := relation_config.config.extra.get("initialize"): + config_dict["initialize"] = initialize.upper() + + return config_dict + + @classmethod + def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: + dynamic_table: "agate.Row" = relation_results["dynamic_table"].rows[0] + + config_dict = { + "name": dynamic_table.get("name"), + "schema_name": dynamic_table.get("schema_name"), + "database_name": dynamic_table.get("database_name"), + "query": dynamic_table.get("text"), + "target_lag": dynamic_table.get("target_lag"), + "snowflake_warehouse": dynamic_table.get("warehouse"), + "catalog": SnowflakeCatalogConfig.parse_relation_results(relation_results), + "refresh_mode": dynamic_table.get("refresh_mode"), + # we don't get initialize since that's a one-time scheduler attribute, not a DT attribute + } + + return config_dict + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeDynamicTableTargetLagConfigChange(RelationConfigChange): + context: Optional[str] = None + + @property + def requires_full_refresh(self) -> bool: + return False + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeDynamicTableWarehouseConfigChange(RelationConfigChange): + context: Optional[str] = None + + @property + def requires_full_refresh(self) -> bool: + return False + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeDynamicTableRefreshModeConfigChange(RelationConfigChange): + context: Optional[str] = None + + @property + def requires_full_refresh(self) -> bool: + return True + + +@dataclass +class SnowflakeDynamicTableConfigChangeset: + target_lag: Optional[SnowflakeDynamicTableTargetLagConfigChange] = None + snowflake_warehouse: Optional[SnowflakeDynamicTableWarehouseConfigChange] = None + refresh_mode: Optional[SnowflakeDynamicTableRefreshModeConfigChange] = None + catalog: Optional[SnowflakeCatalogConfigChange] = None + + @property + def requires_full_refresh(self) -> bool: + return any( + [ + self.target_lag.requires_full_refresh if self.target_lag else False, + ( + self.snowflake_warehouse.requires_full_refresh + if self.snowflake_warehouse + else False + ), + self.refresh_mode.requires_full_refresh if self.refresh_mode else False, + self.catalog.requires_full_refresh if self.catalog else False, + ] + ) + + @property + def has_changes(self) -> bool: + return any([self.target_lag, self.snowflake_warehouse, self.refresh_mode, self.catalog]) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/formats.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/formats.py new file mode 100644 index 000000000..b6bb0bdda --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/formats.py @@ -0,0 +1,19 @@ +from dbt_common.dataclass_schema import StrEnum # doesn't exist in standard library until py3.11 +from typing_extensions import Self + + +class TableFormat(StrEnum): + """ + Snowflake docs refers to this an 'Object Format.' + Data practitioners and interfaces refer to this as 'Table Format's, hence the term's use here. + """ + + DEFAULT = "default" + ICEBERG = "iceberg" + + @classmethod + def default(cls) -> Self: + return cls("default") + + def __str__(self): + return self.value diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/policies.py b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/policies.py new file mode 100644 index 000000000..75195f9a3 --- /dev/null +++ b/dbt-snowflake/src/dbt/adapters/snowflake/relation_configs/policies.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + +from dbt.adapters.base.relation import Policy +from dbt_common.dataclass_schema import StrEnum + + +class SnowflakeRelationType(StrEnum): + Table = "table" + View = "view" + CTE = "cte" + External = "external" + DynamicTable = "dynamic_table" + + +class SnowflakeIncludePolicy(Policy): + database: bool = True + schema: bool = True + identifier: bool = True + + +@dataclass +class SnowflakeQuotePolicy(Policy): + database: bool = False + schema: bool = False + identifier: bool = False diff --git a/dbt-snowflake/src/dbt/include/snowflake/__init__.py b/dbt-snowflake/src/dbt/include/snowflake/__init__.py new file mode 100644 index 000000000..b177e5d49 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/__init__.py @@ -0,0 +1,3 @@ +import os + +PACKAGE_PATH = os.path.dirname(__file__) diff --git a/dbt-snowflake/src/dbt/include/snowflake/dbt_project.yml b/dbt-snowflake/src/dbt/include/snowflake/dbt_project.yml new file mode 100644 index 000000000..fcd2c9a48 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/dbt_project.yml @@ -0,0 +1,5 @@ +config-version: 2 +name: dbt_snowflake +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/adapters.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/adapters.sql new file mode 100644 index 000000000..3c93d41ad --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/adapters.sql @@ -0,0 +1,332 @@ +{% macro get_column_comment_sql(column_name, column_dict) -%} + {% if (column_name|upper in column_dict) -%} + {% set matched_column = column_name|upper -%} + {% elif (column_name|lower in column_dict) -%} + {% set matched_column = column_name|lower -%} + {% elif (column_name in column_dict) -%} + {% set matched_column = column_name -%} + {% else -%} + {% set matched_column = None -%} + {% endif -%} + {% if matched_column -%} + {{ adapter.quote(column_name) }} COMMENT $${{ column_dict[matched_column]['description'] | replace('$', '[$]') }}$$ + {%- else -%} + {{ adapter.quote(column_name) }} COMMENT $$$$ + {%- endif -%} +{% endmacro %} + +{% macro get_persist_docs_column_list(model_columns, query_columns) %} +( + {% for column_name in query_columns %} + {{ get_column_comment_sql(column_name, model_columns) }} + {{- ", " if not loop.last else "" }} + {% endfor %} +) +{% endmacro %} + + +{% macro snowflake__get_columns_in_relation(relation) -%} + {%- set sql -%} + describe table {{ relation.render() }} + {%- endset -%} + {%- set result = run_query(sql) -%} + + {% set maximum = 10000 %} + {% if (result | length) >= maximum %} + {% set msg %} + Too many columns in relation {{ relation.render() }}! dbt can only get + information about relations with fewer than {{ maximum }} columns. + {% endset %} + {% do exceptions.raise_compiler_error(msg) %} + {% endif %} + + {% set columns = [] %} + {% for row in result %} + {% do columns.append(api.Column.from_description(row['name'], row['type'])) %} + {% endfor %} + {% do return(columns) %} +{% endmacro %} + +{% macro snowflake__show_object_metadata(relation) %} + {%- set sql -%} + show objects in {{ relation.include(identifier=False) }} starts with '{{ relation.identifier }}' limit 1 + {%- endset -%} + + {%- set result = run_query(sql) -%} + {{ return(result) }} +{% endmacro %} + +{% macro snowflake__list_schemas(database) -%} + {# 10k limit from here: https://docs.snowflake.net/manuals/sql-reference/sql/show-schemas.html#usage-notes #} + {% set maximum = 10000 %} + {% set sql -%} + show terse schemas in database {{ database }} + limit {{ maximum }} + {%- endset %} + {% set result = run_query(sql) %} + {% if (result | length) >= maximum %} + {% set msg %} + Too many schemas in database {{ database }}! dbt can only get + information about databases with fewer than {{ maximum }} schemas. + {% endset %} + {% do exceptions.raise_compiler_error(msg) %} + {% endif %} + {{ return(result) }} +{% endmacro %} + + +{% macro snowflake__get_paginated_relations_array(max_iter, max_results_per_iter, max_total_results, schema_relation, watermark) %} + + {% set paginated_relations = [] %} + + {% for _ in range(0, max_iter) %} + + {% if schema_relation is string %} + {%- set paginated_sql -%} + show objects in {{ schema_relation }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}' + {%- endset -%} + {% else %} + {%- set paginated_sql -%} + show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}' + {%- endset -%} + {% endif -%} + + {%- set paginated_result = run_query(paginated_sql) %} + {%- set paginated_n = (paginated_result | length) -%} + + {# + terminating condition: if there are 0 records in the result we reached + the end exactly on the previous iteration + #} + {%- if paginated_n == 0 -%} + {%- break -%} + {%- endif -%} + + {# + terminating condition: At some point the user needs to be reasonable with how + many objects are contained in their schemas. Since there was already + one iteration before attempting pagination, loop.index == max_iter means + the limit has been surpassed. + #} + + {%- if loop.index == max_iter -%} + {%- set msg -%} + dbt is currently configured to list a maximum of {{ max_total_results }} objects per schema. + {{ schema_relation }} exceeds this limit. If this is expected, you may configure this limit + by setting list_relations_per_page and list_relations_page_limit in your project flags. + It is recommended to start by increasing list_relations_page_limit to something more than the default of 10. + {%- endset -%} + + {% do exceptions.raise_compiler_error(msg) %} + {%- endif -%} + + {%- do paginated_relations.append(paginated_result) -%} + {% set watermark.table_name = paginated_result.columns[1].values()[-1] %} + + {# + terminating condition: paginated_n < max_results_per_iter means we reached the end + #} + {%- if paginated_n < max_results_per_iter -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + + {{ return(paginated_relations) }} + +{% endmacro %} + +{% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} + + {%- set max_results_per_iter = adapter.config.flags.get('list_relations_per_page', max_results_per_iter) -%} + {%- set max_iter = adapter.config.flags.get('list_relations_page_limit', max_iter) -%} + {%- set max_total_results = max_results_per_iter * max_iter -%} + {%- set sql -%} + {% if schema_relation is string %} + show objects in {{ schema_relation }} limit {{ max_results_per_iter }}; + {% else %} + show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }}; + {% endif -%} + + {# -- Gated for performance reason. If you don't want Iceberg, you shouldn't pay the + -- latency penalty. #} + {% if adapter.behavior.enable_iceberg_materializations.no_warn %} + select all_objects.*, is_iceberg + from table(result_scan(last_query_id(-1))) all_objects + left join INFORMATION_SCHEMA.tables as all_tables + on all_tables.table_name = all_objects."name" + and all_tables.table_schema = all_objects."schema_name" + and all_tables.table_catalog = all_objects."database_name" + {% endif -%} + {%- endset -%} + + {%- set result = run_query(sql) -%} + + {%- set n = (result | length) -%} + {%- set watermark = namespace(table_name=result.columns[1].values()[-1]) -%} + {%- set paginated = namespace(result=[]) -%} + + {% if n >= max_results_per_iter %} + + {% set paginated.result = snowflake__get_paginated_relations_array( + max_iter, + max_results_per_iter, + max_total_results, + schema_relation, + watermark + ) + %} + + {% endif %} + + {%- set all_results_array = [result] + paginated.result -%} + {%- set result = result.merge(all_results_array) -%} + {%- do return(result) -%} + +{% endmacro %} + + +{% macro snowflake__check_schema_exists(information_schema, schema) -%} + {% call statement('check_schema_exists', fetch_result=True) -%} + select count(*) + from {{ information_schema }}.schemata + where upper(schema_name) = upper('{{ schema }}') + and upper(catalog_name) = upper('{{ information_schema.database }}') + {%- endcall %} + {{ return(load_result('check_schema_exists').table) }} +{%- endmacro %} + + +{% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} + {% call statement('alter_column_type') %} + alter {{ relation.get_ddl_prefix_for_alter() }} table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; + {% endcall %} +{% endmacro %} + +{% macro snowflake__alter_relation_comment(relation, relation_comment) -%} + {%- if relation.is_dynamic_table -%} + {%- set relation_type = 'dynamic table' -%} + {%- else -%} + {%- set relation_type = relation.type -%} + {%- endif -%} + comment on {{ relation_type }} {{ relation.render() }} IS $${{ relation_comment | replace('$', '[$]') }}$$; +{% endmacro %} + + +{% macro snowflake__alter_column_comment(relation, column_dict) -%} + {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} + {% if relation.is_dynamic_table -%} + {% set relation_type = "table" %} + {% else -%} + {% set relation_type = relation.type %} + {% endif %} + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} alter + {% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %} + {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }} + {% endfor %} +{% endmacro %} + + +{% macro get_current_query_tag() -%} + {{ return(run_query("show parameters like 'query_tag' in session").rows[0]['value']) }} +{% endmacro %} + + +{% macro set_query_tag() -%} + {{ return(adapter.dispatch('set_query_tag', 'dbt')()) }} +{% endmacro %} + + +{% macro snowflake__set_query_tag() -%} + {% set new_query_tag = config.get('query_tag') %} + {% if new_query_tag %} + {% set original_query_tag = get_current_query_tag() %} + {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }} + {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %} + {{ return(original_query_tag)}} + {% endif %} + {{ return(none)}} +{% endmacro %} + + +{% macro unset_query_tag(original_query_tag) -%} + {{ return(adapter.dispatch('unset_query_tag', 'dbt')(original_query_tag)) }} +{% endmacro %} + + +{% macro snowflake__unset_query_tag(original_query_tag) -%} + {% set new_query_tag = config.get('query_tag') %} + {% if new_query_tag %} + {% if original_query_tag %} + {{ log("Resetting query_tag to '" ~ original_query_tag ~ "'.") }} + {% do run_query("alter session set query_tag = '{}'".format(original_query_tag)) %} + {% else %} + {{ log("No original query_tag, unsetting parameter.") }} + {% do run_query("alter session unset query_tag") %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro snowflake__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} + + {% if relation.is_dynamic_table -%} + {% set relation_type = "dynamic table" %} + {% else -%} + {% set relation_type = relation.type %} + {% endif %} + + {% if add_columns %} + + {% set sql -%} + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} add column + {% for column in add_columns %} + {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {% endfor %} + {%- endset -%} + + {% do run_query(sql) %} + + {% endif %} + + {% if remove_columns %} + + {% set sql -%} + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} drop column + {% for column in remove_columns %} + {{ column.name }}{{ ',' if not loop.last }} + {% endfor %} + {%- endset -%} + + {% do run_query(sql) %} + + {% endif %} + +{% endmacro %} + + + +{% macro snowflake_dml_explicit_transaction(dml) %} + {# + Use this macro to wrap all INSERT, MERGE, UPDATE, DELETE, and TRUNCATE + statements before passing them into run_query(), or calling in the 'main' statement + of a materialization + #} + {% set dml_transaction -%} + begin; + {{ dml }}; + commit; + {%- endset %} + + {% do return(dml_transaction) %} + +{% endmacro %} + + +{% macro snowflake__truncate_relation(relation) -%} + {% set truncate_dml %} + truncate table {{ relation.render() }} + {% endset %} + {% call statement('truncate_relation') -%} + {{ snowflake_dml_explicit_transaction(truncate_dml) }} + {%- endcall %} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/apply_grants.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/apply_grants.sql new file mode 100644 index 000000000..72bea5c1b --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/apply_grants.sql @@ -0,0 +1,8 @@ +{% macro snowflake__copy_grants() %} + {% set copy_grants = config.get('copy_grants', False) %} + {{ return(copy_grants) }} +{% endmacro %} + +{%- macro snowflake__support_multiple_grantees_per_dcl_statement() -%} + {{ return(False) }} +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/catalog.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/catalog.sql new file mode 100644 index 000000000..bde8b8f8f --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/catalog.sql @@ -0,0 +1,131 @@ +{% macro snowflake__get_catalog(information_schema, schemas) -%} + + {% set query %} + with tables as ( + {{ snowflake__get_catalog_tables_sql(information_schema) }} + {{ snowflake__get_catalog_schemas_where_clause_sql(schemas) }} + ), + columns as ( + {{ snowflake__get_catalog_columns_sql(information_schema) }} + {{ snowflake__get_catalog_schemas_where_clause_sql(schemas) }} + ) + {{ snowflake__get_catalog_results_sql() }} + {%- endset -%} + + {{ return(run_query(query)) }} + +{%- endmacro %} + + +{% macro snowflake__get_catalog_relations(information_schema, relations) -%} + + {% set query %} + with tables as ( + {{ snowflake__get_catalog_tables_sql(information_schema) }} + {{ snowflake__get_catalog_relations_where_clause_sql(relations) }} + ), + columns as ( + {{ snowflake__get_catalog_columns_sql(information_schema) }} + {{ snowflake__get_catalog_relations_where_clause_sql(relations) }} + ) + {{ snowflake__get_catalog_results_sql() }} + {%- endset -%} + + {{ return(run_query(query)) }} + +{%- endmacro %} + + +{% macro snowflake__get_catalog_tables_sql(information_schema) -%} + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + case + when is_dynamic = 'YES' and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + else table_type + end as "table_type", + comment as "table_comment", + + -- note: this is the _role_ that owns the table + table_owner as "table_owner", + + 'Clustering Key' as "stats:clustering_key:label", + clustering_key as "stats:clustering_key:value", + 'The key used to cluster this table' as "stats:clustering_key:description", + (clustering_key is not null) as "stats:clustering_key:include", + + 'Row Count' as "stats:row_count:label", + row_count as "stats:row_count:value", + 'An approximate count of rows in this table' as "stats:row_count:description", + (row_count is not null) as "stats:row_count:include", + + 'Approximate Size' as "stats:bytes:label", + bytes as "stats:bytes:value", + 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", + (bytes is not null) as "stats:bytes:include", + + 'Last Modified' as "stats:last_modified:label", + to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + 'The timestamp for last update/change' as "stats:last_modified:description", + (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + from {{ information_schema }}.tables +{%- endmacro %} + + +{% macro snowflake__get_catalog_columns_sql(information_schema) -%} + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + + column_name as "column_name", + ordinal_position as "column_index", + data_type as "column_type", + comment as "column_comment" + from {{ information_schema }}.columns +{%- endmacro %} + + +{% macro snowflake__get_catalog_results_sql() -%} + select * + from tables + join columns using ("table_database", "table_schema", "table_name") + order by "column_index" +{%- endmacro %} + + +{% macro snowflake__catalog_equals(field, value) %} + "{{ field }}" ilike '{{ value }}' and upper("{{ field }}") = upper('{{ value }}') +{% endmacro %} + + +{% macro snowflake__get_catalog_schemas_where_clause_sql(schemas) -%} + where ({%- for schema in schemas -%} + ({{ snowflake__catalog_equals('table_schema', schema) }}){%- if not loop.last %} or {% endif -%} + {%- endfor -%}) +{%- endmacro %} + + +{% macro snowflake__get_catalog_relations_where_clause_sql(relations) -%} + where ( + {%- for relation in relations -%} + {% if relation.schema and relation.identifier %} + ( + {{ snowflake__catalog_equals('table_schema', relation.schema) }} + and {{ snowflake__catalog_equals('table_name', relation.identifier) }} + ) + {% elif relation.schema %} + ( + {{ snowflake__catalog_equals('table_schema', relation.schema) }} + ) + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` requires a list of relations, each with a schema' + ) %} + {% endif %} + + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/clone.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/clone.sql new file mode 100644 index 000000000..665fd593b --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/clone.sql @@ -0,0 +1,11 @@ +{% macro snowflake__can_clone_table() %} + {{ return(True) }} +{% endmacro %} + +{% macro snowflake__create_or_replace_clone(this_relation, defer_relation) %} + create or replace + {{ "transient" if config.get("transient", true) }} + table {{ this_relation }} + clone {{ defer_relation }} + {{ "copy grants" if config.get("copy_grants", false) }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql new file mode 100644 index 000000000..f491ef3bd --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql @@ -0,0 +1,97 @@ +{% materialization dynamic_table, adapter='snowflake' %} + + {% set query_tag = set_query_tag() %} + + {% set existing_relation = load_cached_relation(this) %} + {% set target_relation = this.incorporate(type=this.DynamicTable) %} + + {{ run_hooks(pre_hooks) }} + + {% set build_sql = dynamic_table_get_build_sql(existing_relation, target_relation) %} + + {% if build_sql == '' %} + {{ dynamic_table_execute_no_op(target_relation) }} + {% else %} + {{ dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation) }} + {% endif %} + + {{ run_hooks(post_hooks) }} + + {% do unset_query_tag(query_tag) %} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} + + +{% macro dynamic_table_get_build_sql(existing_relation, target_relation) %} + + {% set full_refresh_mode = should_full_refresh() %} + + -- determine the scenario we're in: create, full_refresh, alter, refresh data + {% if existing_relation is none %} + {% set build_sql = get_create_sql(target_relation, sql) %} + {% elif full_refresh_mode or not existing_relation.is_dynamic_table %} + {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %} + {% else %} + + -- get config options + {% set on_configuration_change = config.get('on_configuration_change') %} + {% set configuration_changes = snowflake__get_dynamic_table_configuration_changes(existing_relation, config) %} + + {% if configuration_changes is none %} + {% set build_sql = '' %} + {{ exceptions.warn("No configuration changes were identified on: `" ~ target_relation ~ "`. Continuing.") }} + + {% elif on_configuration_change == 'apply' %} + {% set build_sql = snowflake__get_alter_dynamic_table_as_sql(existing_relation, configuration_changes, target_relation, sql) %} + {% elif on_configuration_change == 'continue' %} + {% set build_sql = '' %} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} + {% elif on_configuration_change == 'fail' %} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} + + {% else %} + -- this only happens if the user provides a value other than `apply`, 'continue', 'fail' + {{ exceptions.raise_compiler_error("Unexpected configuration scenario: `" ~ on_configuration_change ~ "`") }} + + {% endif %} + + {% endif %} + + {% do return(build_sql) %} + +{% endmacro %} + + +{% macro dynamic_table_execute_no_op(relation) %} + {% do store_raw_result( + name="main", + message="skip " ~ relation, + code="skip", + rows_affected="-1" + ) %} +{% endmacro %} + + +{% macro dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation) %} + + {% set grant_config = config.get('grants') %} + + {% call statement(name="main") %} + {{ build_sql }} + {% endcall %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + +{% endmacro %} + + +{% macro snowflake__get_dynamic_table_configuration_changes(existing_relation, new_config) -%} + {% set _existing_dynamic_table = snowflake__describe_dynamic_table(existing_relation) %} + {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config.model) %} + {% do return(_configuration_changes) %} +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql new file mode 100644 index 000000000..dbb79de02 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql @@ -0,0 +1,170 @@ +{% macro dbt_snowflake_get_tmp_relation_type(strategy, unique_key, language) %} +{%- set tmp_relation_type = config.get('tmp_relation_type') -%} + /* {# + High-level principles: + If we are running multiple statements (DELETE + INSERT), + and we want to guarantee identical inputs to both statements, + then we must first save the model query results as a temporary table + (which presumably comes with a performance cost). + If we are running a single statement (MERGE or INSERT alone), + we _may_ save the model query definition as a view instead, + for (presumably) faster overall incremental processing. + + Low-level specifics: + If an invalid option is specified, then we will raise an + excpetion with corresponding message. + + Languages other than SQL (like Python) will use a temporary table. + With the default strategy of merge, the user may choose between a temporary + table and view (defaulting to view). + + The append strategy can use a view because it will run a single INSERT statement. + + When unique_key is none, the delete+insert and microbatch strategies can use a view beacuse a + single INSERT statement is run with no DELETES as part of the statement. + Otherwise, play it safe by using a temporary table. + #} */ + + {% if language == "python" and tmp_relation_type is not none %} + {% do exceptions.raise_compiler_error( + "Python models currently only support 'table' for tmp_relation_type but " + ~ tmp_relation_type ~ " was specified." + ) %} + {% endif %} + + {% if strategy in ["delete+insert", "microbatch"] and tmp_relation_type is not none and tmp_relation_type != "table" and unique_key is not none %} + {% do exceptions.raise_compiler_error( + "In order to maintain consistent results when `unique_key` is not none, + the `" ~ strategy ~ "` strategy only supports `table` for `tmp_relation_type` but " + ~ tmp_relation_type ~ " was specified." + ) + %} + {% endif %} + + {% if language != "sql" %} + {{ return("table") }} + {% elif tmp_relation_type == "table" %} + {{ return("table") }} + {% elif tmp_relation_type == "view" %} + {{ return("view") }} + {% elif strategy in ("default", "merge", "append") %} + {{ return("view") }} + {% elif strategy in ["delete+insert", "microbatch"] and unique_key is none %} + {{ return("view") }} + {% else %} + {{ return("table") }} + {% endif %} +{% endmacro %} + +{% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%} + + {% set original_query_tag = set_query_tag() %} + + {#-- Set vars --#} + {%- set full_refresh_mode = (should_full_refresh()) -%} + {%- set language = model['language'] -%} + + {%- set identifier = this.name -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, + schema=schema, + database=database, + type='table', + table_format=config.get('table_format', 'default') + ) -%} + + {% set existing_relation = load_relation(this) %} + + {#-- The temp relation will be a view (faster) or temp table, depending on upsert/merge strategy --#} + {%- set unique_key = config.get('unique_key') -%} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set tmp_relation_type = dbt_snowflake_get_tmp_relation_type(incremental_strategy, unique_key, language) %} + {% set tmp_relation = make_temp_relation(this).incorporate(type=tmp_relation_type) %} + + {% set grant_config = config.get('grants') %} + + {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} + + {{ run_hooks(pre_hooks) }} + + {% if existing_relation is none %} + {%- call statement('main', language=language) -%} + {{ create_table_as(False, target_relation, compiled_code, language) }} + {%- endcall -%} + + {% elif existing_relation.is_view %} + {#-- Can't overwrite a view with a table - we must drop --#} + {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} + {% do adapter.drop_relation(existing_relation) %} + {%- call statement('main', language=language) -%} + {{ create_table_as(False, target_relation, compiled_code, language) }} + {%- endcall -%} + + {% elif full_refresh_mode %} + {% if target_relation.needs_to_drop(existing_relation) %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + {%- call statement('main', language=language) -%} + {{ create_table_as(False, target_relation, compiled_code, language) }} + {%- endcall -%} + + {% elif target_relation.table_format != existing_relation.table_format %} + {% do exceptions.raise_compiler_error( + "Unable to alter incremental model `" ~ target_relation.identifier ~ "` to '" ~ target_relation.table_format ~ " table format due to Snowflake limitation. Please execute with --full-refresh to drop the table and recreate in new table format.'" + ) + %} + + {% else %} + {#-- Create the temp relation, either as a view or as a temp table --#} + {% if tmp_relation_type == 'view' %} + {%- call statement('create_tmp_relation') -%} + {{ snowflake__create_view_as_with_temp_flag(tmp_relation, compiled_code, True) }} + {%- endcall -%} + {% else %} + {%- call statement('create_tmp_relation', language=language) -%} + {{ create_table_as(True, tmp_relation, compiled_code, language) }} + {%- endcall -%} + {% endif %} + + {% do adapter.expand_target_column_types( + from_relation=tmp_relation, + to_relation=target_relation) %} + {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} + {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %} + {% if not dest_columns %} + {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} + {% endif %} + + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} + + {%- call statement('main') -%} + {{ strategy_sql_macro_func(strategy_arg_dict) }} + {%- endcall -%} + {% endif %} + + {% do drop_relation_if_exists(tmp_relation) %} + + {{ run_hooks(post_hooks) }} + + {% set target_relation = target_relation.incorporate(type='table') %} + + {% set should_revoke = + should_revoke(existing_relation.is_table, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% do unset_query_tag(original_query_tag) %} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization %} + + +{% macro snowflake__get_incremental_default_sql(arg_dict) %} + {{ return(get_incremental_merge_sql(arg_dict)) }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/merge.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/merge.sql new file mode 100644 index 000000000..7515f6010 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/merge.sql @@ -0,0 +1,81 @@ +{% macro snowflake__get_merge_sql(target, source_sql, unique_key, dest_columns, incremental_predicates) -%} + + {# + Workaround for Snowflake not being happy with a merge on a constant-false predicate. + When no unique_key is provided, this macro will do a regular insert. If a unique_key + is provided, then this macro will do a proper merge instead. + #} + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute='name')) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {%- set dml -%} + {%- if unique_key is none -%} + + {{ sql_header if sql_header is not none }} + + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source_sql }} + ) + + {%- else -%} + + {{ default__get_merge_sql(target, source_sql, unique_key, dest_columns, incremental_predicates) }} + + {%- endif -%} + {%- endset -%} + + {% do return(snowflake_dml_explicit_transaction(dml)) %} + +{% endmacro %} + + +{% macro snowflake__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% set dml = default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% do return(snowflake_dml_explicit_transaction(dml)) %} +{% endmacro %} + + +{% macro snowflake__snapshot_merge_sql(target, source, insert_cols) %} + {% set dml = default__snapshot_merge_sql(target, source, insert_cols) %} + {% do return(snowflake_dml_explicit_transaction(dml)) %} +{% endmacro %} + + +{% macro snowflake__get_incremental_append_sql(get_incremental_append_sql) %} + {% set dml = default__get_incremental_append_sql(get_incremental_append_sql) %} + {% do return(snowflake_dml_explicit_transaction(dml)) %} +{% endmacro %} + + +{% macro snowflake__get_incremental_microbatch_sql(arg_dict) %} + {%- set target = arg_dict["target_relation"] -%} + {%- set source = arg_dict["temp_relation"] -%} + {%- set dest_columns = arg_dict["dest_columns"] -%} + {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} + + {#-- Add additional incremental_predicates to filter for batch --#} + {% if model.batch and model.batch.event_time_start -%} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "')") %} + {% endif %} + {% if model.batch and model.batch.event_time_end -%} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "')") %} + {% endif %} + {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} + + delete from {{ target }} DBT_INTERNAL_TARGET + where ( + {% for predicate in incremental_predicates %} + {%- if not loop.first %}and {% endif -%} {{ predicate }} + {% endfor %} + ); + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/seed.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/seed.sql new file mode 100644 index 000000000..66c7348a9 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/seed.sql @@ -0,0 +1,47 @@ +{% macro snowflake__load_csv_rows(model, agate_table) %} + {% set batch_size = get_batch_size() %} + {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} + {% set bindings = [] %} + + {% set statements = [] %} + + {% for chunk in agate_table.rows | batch(batch_size) %} + {% set bindings = [] %} + + {% for row in chunk %} + {% do bindings.extend(row) %} + {% endfor %} + + {% set sql %} + insert into {{ this.render() }} ({{ cols_sql }}) values + {% for row in chunk -%} + ({%- for column in agate_table.column_names -%} + %s + {%- if not loop.last%},{%- endif %} + {%- endfor -%}) + {%- if not loop.last%},{%- endif %} + {%- endfor %} + {% endset %} + + {% do adapter.add_query('BEGIN', auto_begin=False) %} + {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} + {% do adapter.add_query('COMMIT', auto_begin=False) %} + + {% if loop.index0 == 0 %} + {% do statements.append(sql) %} + {% endif %} + {% endfor %} + + {# Return SQL so we can render it out into the compiled files #} + {{ return(statements[0]) }} +{% endmacro %} + +{% materialization seed, adapter='snowflake' %} + {% set original_query_tag = set_query_tag() %} + + {% set relations = materialization_seed_default() %} + + {% do unset_query_tag(original_query_tag) %} + + {{ return(relations) }} +{% endmaterialization %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/snapshot.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/snapshot.sql new file mode 100644 index 000000000..e79516ba2 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/snapshot.sql @@ -0,0 +1,8 @@ +{% materialization snapshot, adapter='snowflake' %} + {% set original_query_tag = set_query_tag() %} + {% set relations = materialization_snapshot_default() %} + + {% do unset_query_tag(original_query_tag) %} + + {{ return(relations) }} +{% endmaterialization %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql new file mode 100644 index 000000000..995757b6b --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql @@ -0,0 +1,74 @@ +{% materialization table, adapter='snowflake', supported_languages=['sql', 'python']%} + + {% set original_query_tag = set_query_tag() %} + + {%- set identifier = model['alias'] -%} + {%- set language = model['language'] -%} + + {% set grant_config = config.get('grants') %} + + {%- set existing_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, + schema=schema, + database=database, + type='table', + table_format=config.get('table_format', 'default') + ) -%} + + {{ run_hooks(pre_hooks) }} + + {% if target_relation.needs_to_drop(existing_relation) %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + + {% call statement('main', language=language) -%} + {{ create_table_as(False, target_relation, compiled_code, language) }} + {%- endcall %} + + {{ run_hooks(post_hooks) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% do unset_query_tag(original_query_tag) %} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} + +{% macro py_write_table(compiled_code, target_relation, temporary=False, table_type=none) %} +{#- The following logic is only for backwards-compatiblity with deprecated `temporary` parameter -#} +{% if table_type is not none %} + {#- Just use the table_type as-is -#} +{% elif temporary -%} + {#- Case 1 when the deprecated `temporary` parameter is used without the replacement `table_type` parameter -#} + {%- set table_type = "temporary" -%} +{% else %} + {#- Case 2 when the deprecated `temporary` parameter is used without the replacement `table_type` parameter -#} + {#- Snowflake treats "" as meaning "permanent" -#} + {%- set table_type = "" -%} +{%- endif %} +{{ compiled_code }} +def materialize(session, df, target_relation): + # make sure pandas exists + import importlib.util + package_name = 'pandas' + if importlib.util.find_spec(package_name): + import pandas + if isinstance(df, pandas.core.frame.DataFrame): + session.use_database(target_relation.database) + session.use_schema(target_relation.schema) + # session.write_pandas does not have overwrite function + df = session.createDataFrame(df) + {% set target_relation_name = resolve_model_name(target_relation) %} + df.write.mode("overwrite").save_as_table('{{ target_relation_name }}', table_type='{{table_type}}') + +def main(session): + dbt = dbtObj(session.table) + df = model(dbt, session) + materialize(session, df, dbt.this) + return "OK" +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/test.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/test.sql new file mode 100644 index 000000000..816a74893 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/test.sql @@ -0,0 +1,8 @@ +{%- materialization test, adapter='snowflake' -%} + + {% set original_query_tag = set_query_tag() %} + {% set relations = materialization_test_default() %} + {% do unset_query_tag(original_query_tag) %} + {{ return(relations) }} + +{%- endmaterialization -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/view.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/view.sql new file mode 100644 index 000000000..cb306e79f --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/view.sql @@ -0,0 +1,14 @@ +{% materialization view, adapter='snowflake' -%} + + {% set original_query_tag = set_query_tag() %} + {% set to_return = snowflake__create_or_replace_view() %} + + {% set target_relation = this.incorporate(type='view') %} + + {% do persist_docs(target_relation, model, for_columns=false) %} + + {% do unset_query_tag(original_query_tag) %} + + {% do return(to_return) %} + +{%- endmaterialization %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/metadata.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/metadata.sql new file mode 100644 index 000000000..667082fe2 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/metadata.sql @@ -0,0 +1,19 @@ +{% macro snowflake__get_relation_last_modified(information_schema, relations) -%} + + {%- call statement('last_modified', fetch_result=True) -%} + select table_schema as schema, + table_name as identifier, + last_altered as last_modified, + {{ current_timestamp() }} as snapshotted_at + from {{ information_schema }}.tables + where ( + {%- for relation in relations -%} + (upper(table_schema) = upper('{{ relation.schema }}') and + upper(table_name) = upper('{{ relation.identifier }}')){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {%- endcall -%} + + {{ return(load_result('last_modified')) }} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create.sql new file mode 100644 index 000000000..e2cfd38c7 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create.sql @@ -0,0 +1,11 @@ +{% macro snowflake__get_create_sql(relation, sql) %} + + {% if relation.is_dynamic_table %} + {{ snowflake__get_create_dynamic_table_as_sql(relation, sql) }} + + {% else %} + {{ default__get_create_sql(relation, sql) }} + + {% endif %} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create_backup.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create_backup.sql new file mode 100644 index 000000000..b5f347cd9 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/create_backup.sql @@ -0,0 +1,12 @@ +{%- macro snowflake__get_create_backup_sql(relation) -%} + + -- get the standard backup name + {% set backup_relation = make_backup_relation(relation, relation.type) %} + + -- drop any pre-existing backup + {{ get_drop_sql(backup_relation) }}; + + -- use `render` to ensure that the fully qualified name is used + {{ get_rename_sql(relation, backup_relation.render()) }} + +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/drop.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/drop.sql new file mode 100644 index 000000000..99b487461 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/drop.sql @@ -0,0 +1,11 @@ +{% macro snowflake__get_drop_sql(relation) %} + + {% if relation.is_dynamic_table %} + {{ snowflake__get_drop_dynamic_table_sql(relation) }} + + {% else %} + {{ default__get_drop_sql(relation) }} + + {% endif %} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql new file mode 100644 index 000000000..f4b1be699 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql @@ -0,0 +1,25 @@ +{% macro snowflake__get_alter_dynamic_table_as_sql( + existing_relation, + configuration_changes, + target_relation, + sql +) -%} + {{- log('Applying ALTER to: ' ~ existing_relation) -}} + + {% if configuration_changes.requires_full_refresh %} + {{- get_replace_sql(existing_relation, target_relation, sql) -}} + + {% else %} + + {%- set target_lag = configuration_changes.target_lag -%} + {%- if target_lag -%}{{- log('Applying UPDATE TARGET_LAG to: ' ~ existing_relation) -}}{%- endif -%} + {%- set snowflake_warehouse = configuration_changes.snowflake_warehouse -%} + {%- if snowflake_warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%} + + alter dynamic table {{ existing_relation }} set + {% if target_lag %}target_lag = '{{ target_lag.context }}'{% endif %} + {% if snowflake_warehouse %}warehouse = {{ snowflake_warehouse.context }}{% endif %} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/create.sql new file mode 100644 index 000000000..4ebcf145b --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/create.sql @@ -0,0 +1,85 @@ +{% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%} +{#- +-- Produce DDL that creates a dynamic table +-- +-- Args: +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Globals: +-- - config: NodeConfig - contains the attribution required to produce a SnowflakeDynamicTableConfig +-- Returns: +-- A valid DDL statement which will result in a new dynamic table. +-#} + + {%- set dynamic_table = relation.from_config(config.model) -%} + + {%- if dynamic_table.catalog.table_format == 'iceberg' -%} + {{ _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) }} + {%- else -%} + {{ _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) }} + {%- endif -%} + +{%- endmacro %} + + +{% macro _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that creates a standard dynamic table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#syntax +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic standard table. +-#} + + create dynamic table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} + + +{% macro _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that creates a dynamic iceberg table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic iceberg table. +-#} + + create dynamic iceberg table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('external_volume', dynamic_table.catalog.external_volume) }} + {{ optional('catalog', dynamic_table.catalog.name) }} + base_location = '{{ dynamic_table.catalog.base_location }}' + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql new file mode 100644 index 000000000..b5c49ad37 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql @@ -0,0 +1,55 @@ +{% macro snowflake__describe_dynamic_table(relation) %} +{#- +-- Get all relevant metadata about a dynamic table +-- +-- Args: +-- - relation: SnowflakeRelation - the relation to describe +-- Returns: +-- A dictionary with one or two entries depending on whether iceberg is enabled: +-- - dynamic_table: the metadata associated with a standard dynamic table +-- - catalog: the metadata associated with the iceberg catalog +-#} + {%- set _dynamic_table_sql -%} + show dynamic tables + like '{{ relation.identifier }}' + in schema {{ relation.database }}.{{ relation.schema }} + ; + select + "name", + "schema_name", + "database_name", + "text", + "target_lag", + "warehouse", + "refresh_mode" + from table(result_scan(last_query_id())) + {%- endset %} + {% set results = {'dynamic_table': run_query(_dynamic_table_sql)} %} + + {% if adapter.behavior.enable_iceberg_materializations.no_warn %} + {% set _ = results.update({'catalog': run_query(_get_describe_iceberg_catalog_sql(relation))}) %} + {% endif %} + + {% do return(results) %} +{% endmacro %} + + +{% macro _get_describe_iceberg_catalog_sql(relation) %} +{#- +-- Produce DQL that returns all relevant metadata about an iceberg catalog +-- +-- Args: +-- - relation: SnowflakeRelation - the relation to describe +-- Returns: +-- A valid DQL statement that will return metadata associated with an iceberg catalog +-#} + show iceberg tables + like '{{ relation.identifier }}' + in schema {{ relation.database }}.{{ relation.schema }} + ; + select + "catalog_name", + "external_volume_name", + "base_location" + from table(result_scan(last_query_id())) +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql new file mode 100644 index 000000000..577bd06a0 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_drop_dynamic_table_sql(relation) %} + drop dynamic table if exists {{ relation }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql new file mode 100644 index 000000000..5c6af1bda --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql @@ -0,0 +1,5 @@ +{% macro snowflake__refresh_dynamic_table(relation) -%} + {{- log('Applying REFRESH to: ' ~ relation) -}} + + alter dynamic table {{ relation }} refresh +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql new file mode 100644 index 000000000..2e7b4566a --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql @@ -0,0 +1,84 @@ +{% macro snowflake__get_replace_dynamic_table_sql(relation, sql) -%} +{#- +-- Produce DDL that replaces a dynamic table with a new dynamic table +-- +-- Args: +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Globals: +-- - config: NodeConfig - contains the attribution required to produce a SnowflakeDynamicTableConfig +-- Returns: +-- A valid DDL statement which will result in a new dynamic table. +-#} + + {%- set dynamic_table = relation.from_config(config.model) -%} + + {%- if dynamic_table.catalog.table_format == 'iceberg' -%} + {{ _get_replace_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) }} + {%- else -%} + {{ _get_replace_dynamic_standard_table_as_sql(dynamic_table, relation, sql) }} + {%- endif -%} + +{%- endmacro %} + +{% macro _get_replace_dynamic_standard_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that replaces a standard dynamic table with a new standard dynamic table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#syntax +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic standard table. +-#} + + create or replace dynamic table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} + + +{% macro _get_replace_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that replaces a dynamic iceberg table with a new dynamic iceberg table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic iceberg table. +-#} + + create or replace dynamic iceberg table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('external_volume', dynamic_table.catalog.external_volume) }} + {{ optional('catalog', dynamic_table.catalog.name) }} + base_location = '{{ dynamic_table.catalog.base_location }}' + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename.sql new file mode 100644 index 000000000..12f0fae98 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename.sql @@ -0,0 +1,5 @@ +{% macro snowflake__rename_relation(from_relation, to_relation) -%} + {% call statement('rename_relation') -%} + alter table {{ from_relation }} rename to {{ to_relation }} + {%- endcall %} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename_intermediate.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename_intermediate.sql new file mode 100644 index 000000000..abd5fee92 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/rename_intermediate.sql @@ -0,0 +1,9 @@ +{%- macro snowflake__get_rename_intermediate_sql(relation) -%} + + -- get the standard intermediate name + {% set intermediate_relation = make_intermediate_relation(relation) %} + + -- use `render` to ensure that the fully qualified name is used + {{ get_rename_sql(intermediate_relation, relation.render()) }} + +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/replace.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/replace.sql new file mode 100644 index 000000000..1e57c135a --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/replace.sql @@ -0,0 +1,11 @@ +{% macro snowflake__get_replace_sql(existing_relation, target_relation, sql) %} + + {% if existing_relation.is_dynamic_table and target_relation.is_dynamic_table %} + {{ snowflake__get_replace_dynamic_table_sql(target_relation, sql) }} + + {% else %} + {{ default__get_replace_sql(existing_relation, target_relation, sql) }} + + {% endif %} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/create.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/create.sql new file mode 100644 index 000000000..50bedd78f --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/create.sql @@ -0,0 +1,69 @@ +{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} + + {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format') %} + {%- endif %} + + {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} + {%- set alter_prefix = relation.get_ddl_prefix_for_alter() -%} + + {# Generate DDL/DML #} + {%- if language == 'sql' -%} + {%- set cluster_by_keys = config.get('cluster_by', default=none) -%} + {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%} + {%- set copy_grants = config.get('copy_grants', default=false) -%} + + {%- if cluster_by_keys is not none and cluster_by_keys is string -%} + {%- set cluster_by_keys = [cluster_by_keys] -%} + {%- endif -%} + {%- if cluster_by_keys is not none -%} + {%- set cluster_by_string = cluster_by_keys|join(", ")-%} + {% else %} + {%- set cluster_by_string = none -%} + {%- endif -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create or replace {{ materialization_prefix }} table {{ relation }} + {%- if relation.is_iceberg_format %} + {# + Valid DDL in CTAS statements. Plain create statements have a different order. + https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table + #} + {{ relation.get_iceberg_ddl_options(config.model.config) }} + {%- endif -%} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {% set compiled_code = get_select_subquery(compiled_code) %} + {% endif %} + {% if copy_grants and not temporary -%} copy grants {%- endif %} as + ( + {%- if cluster_by_string is not none -%} + select * from ( + {{ compiled_code }} + ) order by ({{ cluster_by_string }}) + {%- else -%} + {{ compiled_code }} + {%- endif %} + ); + {% if cluster_by_string is not none and not temporary -%} + alter {{ alter_prefix }} table {{relation}} cluster by ({{cluster_by_string}}); + {%- endif -%} + {% if enable_automatic_clustering and cluster_by_string is not none and not temporary %} + alter {{ alter_prefix }} table {{relation}} resume recluster; + {%- endif -%} + + {%- elif language == 'python' -%} + {%- if relation.is_iceberg_format %} + {% do exceptions.raise_compiler_error('Iceberg is incompatible with Python models. Please use a SQL model for the iceberg format.') %} + {%- endif %} + {{ py_write_table(compiled_code=compiled_code, target_relation=relation, table_type=relation.get_ddl_prefix_for_create(config.model.config, temporary)) }} + {%- else -%} + {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %} + {%- endif -%} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/drop.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/drop.sql new file mode 100644 index 000000000..9799d3998 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/drop.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_drop_table_sql(relation) %} + drop table if exists {{ relation }} cascade +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/rename.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/rename.sql new file mode 100644 index 000000000..699debf28 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/rename.sql @@ -0,0 +1,13 @@ +{%- macro snowflake__get_rename_table_sql(relation, new_name) -%} + /* + Rename or move a table to the new name. + + Args: + relation: SnowflakeRelation - relation to be renamed + new_name: Union[str, SnowflakeRelation] - new name for `relation` + if providing a string, the default database/schema will be used if that string is just an identifier + if providing a SnowflakeRelation, `render` will be used to produce a fully qualified name + Returns: templated string + */ + alter table {{ relation }} rename to {{ new_name }} +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/replace.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/replace.sql new file mode 100644 index 000000000..6b93e2ab5 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/table/replace.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_replace_table_sql(relation, sql) %} + {{ snowflake__create_table_as(False, relation, sql) }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql new file mode 100644 index 000000000..d518ac3e1 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql @@ -0,0 +1,73 @@ +{% macro snowflake__create_view_as_with_temp_flag(relation, sql, is_temporary=False) -%} + {%- set secure = config.get('secure', default=false) -%} + {%- set copy_grants = config.get('copy_grants', default=false) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create or replace {% if secure -%} + secure + {%- endif %} {% if is_temporary -%} + temporary + {%- endif %} view {{ relation }} + {% if config.persist_column_docs() -%} + {% set model_columns = model.columns %} + {% set query_columns = get_columns_in_query(sql) %} + {{ get_persist_docs_column_list(model_columns, query_columns) }} + + {%- endif %} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + {% if copy_grants -%} copy grants {%- endif %} as ( + {{ sql }} + ); +{% endmacro %} + + +{% macro snowflake__create_view_as(relation, sql) -%} + {{ snowflake__create_view_as_with_temp_flag(relation, sql) }} +{% endmacro %} + + +/* {# +Vendored from dbt-core for the purpose of overwriting small pieces to support dynamics tables. This should +eventually be retired in favor of a standardized approach. Changed line: + +{%- if old_relation is not none and old_relation.is_table -%} -> +{%- if old_relation is not none and not old_relation.is_view -%} +#} */ + +{% macro snowflake__create_or_replace_view() %} + {%- set identifier = model['alias'] -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, database=database, + type='view') -%} + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks) }} + + -- If there's a table with the same name and we weren't told to full refresh, + -- that's an error. If we were told to full refresh, drop it. This behavior differs + -- for Snowflake and BigQuery, so multiple dispatch is used. + {%- if old_relation is not none and not old_relation.is_view -%} + {{ handle_existing_table(should_full_refresh(), old_relation) }} + {%- endif -%} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(target_relation, sql) }} + {%- endcall %} + + {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {{ run_hooks(post_hooks) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/drop.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/drop.sql new file mode 100644 index 000000000..708ca3386 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/drop.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_drop_view_sql(relation) %} + drop view if exists {{ relation }} cascade +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/rename.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/rename.sql new file mode 100644 index 000000000..add2f49b9 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/rename.sql @@ -0,0 +1,13 @@ +{%- macro snowflake__get_rename_view_sql(relation, new_name) -%} + /* + Rename or move a view to the new name. + + Args: + relation: SnowflakeRelation - relation to be renamed + new_name: Union[str, SnowflakeRelation] - new name for `relation` + if providing a string, the default database/schema will be used if that string is just an identifier + if providing a SnowflakeRelation, `render` will be used to produce a fully qualified name + Returns: templated string + */ + alter view {{ relation }} rename to {{ new_name }} +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/replace.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/replace.sql new file mode 100644 index 000000000..a46898f01 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/replace.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_replace_view_sql(relation, sql) %} + {{ snowflake__create_view_as(relation, sql) }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/array_construct.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/array_construct.sql new file mode 100644 index 000000000..90db28e00 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/array_construct.sql @@ -0,0 +1,3 @@ +{% macro snowflake__array_construct(inputs, data_type) -%} + array_construct( {{ inputs|join(' , ') }} ) +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/bool_or.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/bool_or.sql new file mode 100644 index 000000000..a8667ee8d --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/bool_or.sql @@ -0,0 +1,5 @@ +{% macro snowflake__bool_or(expression) -%} + + boolor_agg({{ expression }}) + +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/cast.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/cast.sql new file mode 100644 index 000000000..3da7c3aec --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/cast.sql @@ -0,0 +1,9 @@ +{% macro snowflake__cast(field, type) %} + {% if (type|upper == "GEOGRAPHY") -%} + to_geography({{field}}) + {% elif (type|upper == "GEOMETRY") -%} + to_geometry({{field}}) + {% else -%} + cast({{field}} as {{type}}) + {% endif -%} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/escape_single_quotes.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/escape_single_quotes.sql new file mode 100644 index 000000000..0e5f3c809 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/escape_single_quotes.sql @@ -0,0 +1,4 @@ +{# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} +{% macro snowflake__escape_single_quotes(expression) -%} +{{ expression | replace("'", "\\'") }} +{%- endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/optional.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/optional.sql new file mode 100644 index 000000000..0758ca59f --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/optional.sql @@ -0,0 +1,14 @@ +{% macro optional(name, value, quote_char = '') %} +{#- +-- Insert optional DDL parameters only when their value is provided; makes DDL statements more readable +-- +-- Args: +-- - name: the name of the DDL option +-- - value: the value of the DDL option, may be None +-- - quote_char: the quote character to use (e.g. string), leave blank if unnecessary (e.g. integer or bool) +-- Returns: +-- If the value is not None (e.g. provided by the user), return the option setting DDL +-- If the value is None, return an empty string +-#} +{% if value is not none %}{{ name }} = {{ quote_char }}{{ value }}{{ quote_char }}{% endif %} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/right.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/right.sql new file mode 100644 index 000000000..575e11480 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/right.sql @@ -0,0 +1,12 @@ +{% macro snowflake__right(string_text, length_expression) %} + + case when {{ length_expression }} = 0 + then '' + else + right( + {{ string_text }}, + {{ length_expression }} + ) + end + +{%- endmacro -%} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/safe_cast.sql new file mode 100644 index 000000000..6ff4e351e --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -0,0 +1,13 @@ +{% macro snowflake__safe_cast(field, type) %} + {% if type|upper == "GEOMETRY" -%} + try_to_geometry({{field}}) + {% elif type|upper == "GEOGRAPHY" -%} + try_to_geography({{field}}) + {% elif type|upper != "VARIANT" -%} + {#-- Snowflake try_cast does not support casting to variant, and expects the field as a string --#} + {% set field_as_string = dbt.string_literal(field) if field is number else field %} + try_cast({{field_as_string}} as {{type}}) + {% else -%} + {{ adapter.dispatch('cast', 'dbt')(field, type) }} + {% endif -%} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/utils/timestamps.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/timestamps.sql new file mode 100644 index 000000000..ecbc9940b --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/utils/timestamps.sql @@ -0,0 +1,20 @@ +{% macro snowflake__current_timestamp() -%} + convert_timezone('UTC', current_timestamp()) +{%- endmacro %} + +{% macro snowflake__snapshot_string_as_time(timestamp) -%} + {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} + {{ return(result) }} +{%- endmacro %} + +{% macro snowflake__snapshot_get_time() -%} + to_timestamp_ntz({{ current_timestamp() }}) +{%- endmacro %} + +{% macro snowflake__current_timestamp_backcompat() %} + current_timestamp::{{ type_timestamp() }} +{% endmacro %} + +{% macro snowflake__current_timestamp_in_utc_backcompat() %} + convert_timezone('UTC', {{ snowflake__current_timestamp_backcompat() }})::{{ type_timestamp() }} +{% endmacro %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/profile_template.yml b/dbt-snowflake/src/dbt/include/snowflake/profile_template.yml new file mode 100644 index 000000000..b437853e7 --- /dev/null +++ b/dbt-snowflake/src/dbt/include/snowflake/profile_template.yml @@ -0,0 +1,34 @@ +fixed: + type: snowflake +prompts: + account: + hint: 'https://.snowflakecomputing.com' + user: + hint: 'dev username' + _choose_authentication_type: + password: + password: + hint: 'dev password' + hide_input: true + keypair: + private_key_path: + hint: 'path/to/private.key' + private_key_passphrase: + hint: 'passphrase for the private key, if key is encrypted' + hide_input: true + sso: + authenticator: + hint: "'externalbrowser' or a valid Okta URL" + default: 'externalbrowser' + role: + hint: 'dev role' + warehouse: + hint: 'warehouse name' + database: + hint: 'default database that dbt will build objects in' + schema: + hint: 'default schema that dbt will build objects in' + threads: + hint: '1 or more' + type: 'int' + default: 1 diff --git a/dbt-snowflake/test.env.example b/dbt-snowflake/test.env.example new file mode 100644 index 000000000..bdf5d68e1 --- /dev/null +++ b/dbt-snowflake/test.env.example @@ -0,0 +1,35 @@ +# Note: Make sure you have a Snowflake account that is set up so these fields are easy to complete. +# If you don't have an account set up yet, then take note of these required fields that way. When you're getting set up, +# you can use them later to build your Snowflake project. + +### Test Environment field definitions +# These will all be gathered from account information or created by you. + +# SNOWFLAKE_TEST_ACCOUNT: The name that uniquely identifies your Snowflake account. +# SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database. +# SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing. +# SNOWFLAKE_TEST_DATABASE: Name of the primary database to use for testing. +# SNOWFLAKE_TEST_OAUTH_CLIENT_ID: Client ID of the OAuth client integration. (only for oauth authentication) +# SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: Client secret of your OAuth client id. (only for oauth authentication) +# SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: Boolean value defaulted to True keep connection alive. (only for oauth authentication) +# SNOWFLAKE_TEST_PASSWORD:Password used for your database user. +# SNOWFLAKE_TEST_QUOTED_DATABASE: Name of database to be used from warehouse. +# SNOWFLAKE_TEST_USER: Username of database user +# SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary. + +# Copy the following to a test.env, and replace example values with your information. +SNOWFLAKE_TEST_ACCOUNT=my_account_id +SNOWFLAKE_TEST_ALT_DATABASE=my_alt_database_name +SNOWFLAKE_TEST_ALT_WAREHOUSE=my_alt_warehouse_name +SNOWFLAKE_TEST_DATABASE=my_database_name +SNOWFLAKE_TEST_OAUTH_CLIENT_ID=my_oauth_id +SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=my_oauth_secret +SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=TRUE +SNOWFLAKE_TEST_PASSWORD=my_password +SNOWFLAKE_TEST_QUOTED_DATABASE=my_quoted_database_name +SNOWFLAKE_TEST_USER=my_username +SNOWFLAKE_TEST_WAREHOUSE=my_warehouse_name + +DBT_TEST_USER_1=dbt_test_role_1 +DBT_TEST_USER_2=dbt_test_role_2 +DBT_TEST_USER_3=dbt_test_role_3 diff --git a/dbt-snowflake/tests/__init__.py b/dbt-snowflake/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/conftest.py b/dbt-snowflake/tests/conftest.py new file mode 100644 index 000000000..9993905ff --- /dev/null +++ b/dbt-snowflake/tests/conftest.py @@ -0,0 +1,21 @@ +import pytest +import os + +# Import the fuctional fixtures as a plugin +# Note: fixtures with session scope need to be local + +pytest_plugins = ["dbt.tests.fixtures.project"] + + +# The profile dictionary, used to write out profiles.yml +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "user": os.getenv("SNOWFLAKE_TEST_USER"), + "password": os.getenv("SNOWFLAKE_TEST_PASSWORD"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + } diff --git a/dbt-snowflake/tests/functional/adapter/catalog_tests/files.py b/dbt-snowflake/tests/functional/adapter/catalog_tests/files.py new file mode 100644 index 000000000..a6759398e --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/catalog_tests/files.py @@ -0,0 +1,32 @@ +MY_SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='30 minutes', +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/dbt-snowflake/tests/functional/adapter/catalog_tests/test_relation_types.py b/dbt-snowflake/tests/functional/adapter/catalog_tests/test_relation_types.py new file mode 100644 index 000000000..27267a1c9 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/catalog_tests/test_relation_types.py @@ -0,0 +1,44 @@ +from dbt.contracts.results import CatalogArtifact +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.catalog_tests import files + + +class TestCatalogRelationTypes: + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": files.MY_TABLE, + "my_view.sql": files.MY_VIEW, + "my_dynamic_table.sql": files.MY_DYNAMIC_TABLE, + } + + @pytest.fixture(scope="class", autouse=True) + def docs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield run_dbt(["docs", "generate"]) + + @pytest.mark.parametrize( + "node_name,relation_type", + [ + ("seed.test.my_seed", "BASE TABLE"), + ("model.test.my_table", "BASE TABLE"), + ("model.test.my_view", "VIEW"), + ("model.test.my_dynamic_table", "DYNAMIC TABLE"), + ], + ) + def test_relation_types_populate_correctly( + self, docs: CatalogArtifact, node_name: str, relation_type: str + ): + """ + This test addresses: https://github.com/dbt-labs/dbt-snowflake/issues/817 + """ + assert node_name in docs.nodes + node = docs.nodes[node_name] + assert node.metadata.type == relation_type diff --git a/dbt-snowflake/tests/functional/adapter/column_types/fixtures.py b/dbt-snowflake/tests/functional/adapter/column_types/fixtures.py new file mode 100644 index 000000000..64b88111f --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/column_types/fixtures.py @@ -0,0 +1,46 @@ +_MODEL_SQL = """ +select + 1::smallint as smallint_col, + 2::int as int_col, + 3::bigint as bigint_col, + 4::integer as integer_col, + 5::tinyint as tinyint_col, + 6::byteint as byteint_col, + 7.0::float as float_col, + 8.0::float4 as float4_col, + 9.0::float8 as float8_col, + 10.0::double as double_col, + 11.0::double precision as double_p_col, + 12.0::real as real_col, + 13.0::numeric as numeric_col, + 14.0::decimal as decimal_col, + 15.0::number as number_col, + '16'::text as text_col, + '17'::varchar(20) as varchar_col +""" + +_SCHEMA_YML = """ +version: 2 +models: + - name: model + data_tests: + - is_type: + column_map: + smallint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + int_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + bigint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + integer_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + tinyint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + byteint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + float_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + float4_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + float8_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + double_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + double_p_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + real_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] + numeric_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + decimal_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + number_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] + text_col: ['string', 'not number'] + varchar_col: ['string', 'not number'] +""" diff --git a/dbt-snowflake/tests/functional/adapter/column_types/test_column_types.py b/dbt-snowflake/tests/functional/adapter/column_types/test_column_types.py new file mode 100644 index 000000000..0bc364d57 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/column_types/test_column_types.py @@ -0,0 +1,12 @@ +import pytest +from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes +from tests.functional.adapter.column_types.fixtures import _MODEL_SQL, _SCHEMA_YML + + +class TestSnowflakeColumnTypes(BaseColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": _MODEL_SQL, "schema.yml": _SCHEMA_YML} + + def test_run_and_test(self, project): + self.run_and_test() diff --git a/dbt-snowflake/tests/functional/adapter/custom_schema_tests/seeds.py b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/seeds.py new file mode 100644 index 000000000..2fe1fac4e --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/seeds.py @@ -0,0 +1,13 @@ +seed_csv = """ +id,first_name,last_name,email,gender,ip_address +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +""".lstrip() + +seed_agg_csv = """ +last_name,count +Hunter,2 +Walker,2 +Ryan,2 +""".lstrip() diff --git a/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_database.py b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_database.py new file mode 100644 index 000000000..00c2e7c80 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_database.py @@ -0,0 +1,123 @@ +import pytest +import os +from dbt.tests.util import check_relations_equal, check_table_does_exist, run_dbt +from tests.functional.adapter.custom_schema_tests.seeds import seed_agg_csv, seed_csv + +_VIEW_1_SQL = """ +select * from {{ ref('seed') }} +""".lstrip() + +_VIEW_2_SQL = """ +{{ config(database='alt') }} +select * from {{ ref('view_1') }} +""".lstrip() + +_VIEW_3_SQL = """ +{{ config(database='alt', materialized='table') }} + + +with v1 as ( + + select * from {{ ref('view_1') }} + +), + +v2 as ( + + select * from {{ ref('view_2') }} + +), + +combined as ( + + select last_name from v1 + union all + select last_name from v2 + +) + +select + last_name, + count(*) as count + +from combined +group by 1 +""".lstrip() + +_CUSTOM_DB_SQL = """ +{% macro generate_database_name(database_name, node) %} + {% if database_name == 'alt' %} + {{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }} + {% elif database_name %} + {{ database_name }} + {% else %} + {{ target.database }} + {% endif %} +{% endmacro %} +""".lstrip() + +ALT_DATABASE = os.getenv("SNOWFLAKE_TEST_ALT_DATABASE") + + +class TestOverrideDatabase: + @pytest.fixture(scope="class") + def macros(self): + return { + "custom_db.sql": _CUSTOM_DB_SQL, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seed_csv, "agg.csv": seed_agg_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "view_1.sql": _VIEW_1_SQL, + "view_2.sql": _VIEW_2_SQL, + "view_3.sql": _VIEW_3_SQL, + } + + @pytest.fixture(scope="function") + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=ALT_DATABASE, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + def test_snowflake_override_generate_db_name(self, project, clean_up): + seed_results = run_dbt(["seed", "--full-refresh"]) + assert len(seed_results) == 2 + + db_with_schema = f"{project.database}.{project.test_schema}" + alt_db_with_schema = f"{ALT_DATABASE}.{project.test_schema}" + seed_table = "SEED" + agg_table = "AGG" + view_1 = "VIEW_1" + view_2 = "VIEW_2" + view_3 = "VIEW_3" + + check_table_does_exist(project.adapter, f"{db_with_schema}.{seed_table}") + check_table_does_exist(project.adapter, f"{db_with_schema}.{agg_table}") + + results = run_dbt() + assert len(results) == 3 + + check_table_does_exist(project.adapter, f"{db_with_schema}.{view_1}") + check_table_does_exist(project.adapter, f"{alt_db_with_schema}.{view_2}") + check_table_does_exist(project.adapter, f"{alt_db_with_schema}.{view_3}") + + # not overridden + check_relations_equal( + project.adapter, [f"{db_with_schema}.{seed_table}", f"{db_with_schema}.{view_1}"] + ) + + # overridden + check_relations_equal( + project.adapter, [f"{db_with_schema}.{seed_table}", f"{alt_db_with_schema}.{view_2}"] + ) + check_relations_equal( + project.adapter, [f"{db_with_schema}.{agg_table}", f"{alt_db_with_schema}.{view_3}"] + ) diff --git a/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_schema.py b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_schema.py new file mode 100644 index 000000000..bd7cb502a --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/custom_schema_tests/test_custom_schema.py @@ -0,0 +1,88 @@ +import pytest +from dbt.tests.util import check_relations_equal, run_dbt +from tests.functional.adapter.custom_schema_tests.seeds import seed_agg_csv, seed_csv + +_VIEW_1_SQL = """ +select * from {{ ref('seed') }} +""".lstrip() + +_VIEW_2_SQL = """ +{{ config(schema='custom') }} + +select * from {{ ref('view_1') }} +""".lstrip() + +_VIEW_3_SQL = """ +{{ config(schema='test', materialized='table') }} + + +with v1 as ( + + select * from {{ ref('view_1') }} + +), + +v2 as ( + + select * from {{ ref('view_2') }} + +), + +combined as ( + + select last_name from v1 + union all + select last_name from v2 + +) + +select + last_name, + count(*) as count + +from combined +group by 1 +""".lstrip() + + +class TestCustomProjectSchemaWithPrefix: + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seed_csv, "agg.csv": seed_agg_csv} + + @pytest.fixture(scope="class") + def models(self): + return {"view_1.sql": _VIEW_1_SQL, "view_2.sql": _VIEW_2_SQL, "view_3.sql": _VIEW_3_SQL} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"schema": "dbt_test"}} + + @pytest.fixture(scope="function") + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + alt_schema_list = ["DBT_TEST", "CUSTOM", "TEST"] + for alt_schema in alt_schema_list: + alt_test_schema = f"{project.test_schema}_{alt_schema}" + relation = project.adapter.Relation.create( + database=project.database, schema=alt_test_schema + ) + project.adapter.drop_schema(relation) + + def test__snowflake__custom_schema_with_prefix(self, project, clean_up): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 2 + results = run_dbt() + assert len(results) == 3 + + db_with_schema = f"{project.database}.{project.test_schema}" + check_relations_equal( + project.adapter, [f"{db_with_schema}.SEED", f"{db_with_schema}_DBT_TEST.VIEW_1"] + ) + check_relations_equal( + project.adapter, [f"{db_with_schema}.SEED", f"{db_with_schema}_CUSTOM.VIEW_2"] + ) + check_relations_equal( + project.adapter, [f"{db_with_schema}.AGG", f"{db_with_schema}_TEST.VIEW_3"] + ) diff --git a/dbt-snowflake/tests/functional/adapter/dbt_clone/test_dbt_clone.py b/dbt-snowflake/tests/functional/adapter/dbt_clone/test_dbt_clone.py new file mode 100644 index 000000000..2a73eb7f4 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/dbt_clone/test_dbt_clone.py @@ -0,0 +1,82 @@ +import pytest +import shutil +import os +from copy import deepcopy +from dbt.tests.util import run_dbt +from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseClonePossible + + +class TestSnowflakeClonePossible(BaseClonePossible): + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_SEEDS" + ) + project.adapter.drop_schema(relation) + + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + pass + + +table_model_1_sql = """ + {{ config( + materialized='table', + transient=true, + ) }} + + select 1 as fun + """ + + +class TestSnowflakeCloneTrainsentTable: + @pytest.fixture(scope="class") + def models(self): + return { + "table_model.sql": table_model_1_sql, + } + + @pytest.fixture(scope="class") + def other_schema(self, unique_schema): + return unique_schema + "_other" + + @pytest.fixture(scope="class") + def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): + outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} + outputs["default"]["schema"] = unique_schema + outputs["otherschema"]["schema"] = other_schema + return {"test": {"outputs": outputs, "target": "default"}} + + def copy_state(self, project_root): + state_path = os.path.join(project_root, "state") + if not os.path.exists(state_path): + os.makedirs(state_path) + shutil.copyfile( + f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" + ) + + def run_and_save_state(self, project_root, with_snapshot=False): + results = run_dbt(["run"]) + assert len(results) == 1 + + self.copy_state(project_root) + + def test_can_clone_transient_table(self, project, other_schema): + project.create_test_schema(other_schema) + self.run_and_save_state(project.project_root) + + clone_args = [ + "clone", + "--state", + "state", + "--target", + "otherschema", + ] + + results = run_dbt(clone_args) + assert len(results) == 1 diff --git a/dbt-snowflake/tests/functional/adapter/dbt_show/test_dbt_show.py b/dbt-snowflake/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 000000000..d12b91f52 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,17 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowSqlHeader, + BaseShowLimit, + BaseShowDoesNotHandleDoubleLimit, +) + + +class TestSnowflakeShowLimit(BaseShowLimit): + pass + + +class TestSnowflakeShowSqlHeader(BaseShowSqlHeader): + pass + + +class TestSnowflakeShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + DATABASE_ERROR_MESSAGE = "unexpected 'limit'" diff --git a/dbt-snowflake/tests/functional/adapter/empty/test_empty.py b/dbt-snowflake/tests/functional/adapter/empty/test_empty.py new file mode 100644 index 000000000..0bf9d1a41 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/empty/test_empty.py @@ -0,0 +1,17 @@ +from dbt.tests.adapter.empty.test_empty import ( + BaseTestEmpty, + BaseTestEmptyInlineSourceRef, + MetadataWithEmptyFlag, +) + + +class TestSnowflakeEmpty(BaseTestEmpty): + pass + + +class TestSnowflakeEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + pass + + +class TestMetadataWithEmptyFlag(MetadataWithEmptyFlag): + pass diff --git a/dbt-snowflake/tests/functional/adapter/expected_stats.py b/dbt-snowflake/tests/functional/adapter/expected_stats.py new file mode 100644 index 000000000..d5e582631 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/expected_stats.py @@ -0,0 +1,34 @@ +from dbt.tests.util import AnyString, AnyInteger + + +def snowflake_stats(): + return { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": True, + "description": "Indicates whether there are statistics for this table", + "include": False, + }, + "bytes": { + "id": "bytes", + "label": "Approximate Size", + "value": AnyInteger(), + "description": "Approximate size of the table as reported by Snowflake", + "include": True, + }, + "last_modified": { + "id": "last_modified", + "label": "Last Modified", + "value": AnyString(), + "description": "The timestamp for last update/change", + "include": True, + }, + "row_count": { + "id": "row_count", + "label": "Row Count", + "value": 1.0, + "description": "An approximate count of rows in this table", + "include": True, + }, + } diff --git a/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py new file mode 100644 index 000000000..022ebca07 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + + +class TestMergeExcludeColumns(BaseMergeExcludeColumns): + pass diff --git a/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_on_schema_change.py new file mode 100644 index 000000000..7b73d212b --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( + BaseIncrementalOnSchemaChange, +) + + +class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): + pass diff --git a/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_predicates.py b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_predicates.py new file mode 100644 index 000000000..ad2c793eb --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_predicates.py @@ -0,0 +1,34 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates + + +class TestIncrementalPredicatesDeleteInsertSnowflake(BaseIncrementalPredicates): + pass + + +class TestPredicatesDeleteInsertSnowflake(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"+predicates": ["id != 2"], "+incremental_strategy": "delete+insert"}} + + +class TestIncrementalPredicatesMergeSnowflake(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_predicates": ["dbt_internal_dest.id != 2"], + "+incremental_strategy": "merge", + } + } + + +class TestPredicatesMergeSnowflake(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+predicates": ["dbt_internal_dest.id != 2"], + "+incremental_strategy": "merge", + } + } diff --git a/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_run_result.py b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_run_result.py new file mode 100644 index 000000000..c4c278273 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_run_result.py @@ -0,0 +1,29 @@ +from dbt.tests.util import run_dbt +from dbt.tests.adapter.basic.test_incremental import ( + BaseIncremental, + BaseIncrementalNotSchemaChange, +) + + +class TestBaseIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): + pass + + +class TestIncrementalRunResultSnowflake(BaseIncremental): + """Bonus test to verify that incremental models return the number of rows affected""" + + def test_incremental(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 2 + + # run with initial seed + results = run_dbt(["run", "--vars", "seed_name: base"]) + assert len(results) == 1 + + # run with additions + results = run_dbt(["run", "--vars", "seed_name: added"]) + assert len(results) == 1 + # verify that run_result is correct + rows_affected = results[0].adapter_response["rows_affected"] + assert rows_affected == 10, f"Expected 10 rows changed, found {rows_affected}" diff --git a/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_unique_id.py new file mode 100644 index 000000000..45b62786c --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -0,0 +1,12 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey + + +class TestUniqueKeySnowflake(BaseIncrementalUniqueKey): + pass + + +class TestUniqueKeyDeleteInsertSnowflake(BaseIncrementalUniqueKey): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"+incremental_strategy": "delete+insert"}} diff --git a/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_pagination.py b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_pagination.py new file mode 100644 index 000000000..7dd382af5 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_pagination.py @@ -0,0 +1,111 @@ +import os + +import pytest + +from dbt_common.exceptions import CompilationError +from dbt.tests.util import run_dbt + +""" +Testing rationale: +- snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call +- when dbt attempts to write into a schema with more than 10K objects, compilation will fail + unless we paginate the result +- we default pagination to 10 pages, but users want to configure this + - we instead use that here to force failures by making it smaller +""" + + +TABLE = """ +{{ config(materialized='table') }} +select 1 as id +""" + + +VIEW = """ +{{ config(materialized='view') }} +select id from {{ ref('my_model_base') }} +""" + + +DYNAMIC_TABLE = ( + """ +{{ config( + materialized='dynamic_table', + target_lag='1 hour', + snowflake_warehouse='""" + + os.getenv("SNOWFLAKE_TEST_WAREHOUSE") + + """', +) }} + +select id from {{ ref('my_model_base') }} +""" +) + + +class BaseConfig: + VIEWS = 90 + DYNAMIC_TABLES = 10 + + @pytest.fixture(scope="class") + def models(self): + my_models = {"my_model_base.sql": TABLE} + for view in range(0, self.VIEWS): + my_models[f"my_model_{view}.sql"] = VIEW + for dynamic_table in range(0, self.DYNAMIC_TABLES): + my_models[f"my_dynamic_table_{dynamic_table}.sql"] = DYNAMIC_TABLE + return my_models + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["run"]) + + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + relations = project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs + ) + assert len(relations) == self.VIEWS + self.DYNAMIC_TABLES + 1 + + +class TestListRelationsWithoutCachingSmall(BaseConfig): + pass + + +class TestListRelationsWithoutCachingLarge(BaseConfig): + @pytest.fixture(scope="class") + def profiles_config_update(self): + return { + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 20, + } + } + + +class TestListRelationsWithoutCachingTooLarge(BaseConfig): + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 5, + } + } + + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + with pytest.raises(CompilationError) as error: + project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs + ) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg + + def test_on_run(self, project): + with pytest.raises(CompilationError) as error: + run_dbt(["run"]) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg diff --git a/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_show_objects.py b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_show_objects.py new file mode 100644 index 000000000..91fb94f79 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_show_objects.py @@ -0,0 +1,127 @@ +import os +from typing import List + +import pytest + +from pathlib import Path + +from dbt.adapters.factory import get_adapter_by_type +from dbt.adapters.snowflake import SnowflakeRelation + +from dbt.tests.util import run_dbt, get_connection + + +SEED = """ +id,value +0,red +1,yellow +2,blue +""".strip() + + +VIEW = """ +select * from {{ ref('my_seed') }} +""" + + +TABLE = """ +{{ config(materialized='table') }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE = ( + """ +{{ config( + materialized='dynamic_table', + target_lag='1 day', + snowflake_warehouse='""" + + os.getenv("SNOWFLAKE_TEST_WAREHOUSE") + + """', +) }} +select * from {{ ref('my_seed') }} +""" +) + +_MODEL_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + ) +}} + +select 1 +""" + + +class ShowObjectsBase: + @staticmethod + def list_relations_without_caching(project) -> List[SnowflakeRelation]: + my_adapter = get_adapter_by_type("snowflake") + schema = my_adapter.Relation.create( + database=project.database, schema=project.test_schema, identifier="" + ) + with get_connection(my_adapter): + relations = my_adapter.list_relations_without_caching(schema) + return relations + + +class TestShowObjects(ShowObjectsBase): + views: int = 10 + tables: int = 10 + dynamic_tables: int = 10 + + @pytest.fixture(scope="class") + def seeds(self): + yield {"my_seed.csv": SEED} + + @pytest.fixture(scope="class") + def models(self): + models = {} + models.update({f"my_view_{i}.sql": VIEW for i in range(self.views)}) + models.update({f"my_table_{i}.sql": TABLE for i in range(self.tables)}) + models.update( + {f"my_dynamic_table_{i}.sql": DYNAMIC_TABLE for i in range(self.dynamic_tables)} + ) + yield models + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + def test_list_relations_without_caching(self, project): + relations = self.list_relations_without_caching(project) + assert len([relation for relation in relations if relation.is_view]) == self.views + assert ( + len([relation for relation in relations if relation.is_table]) + == self.tables + 1 # add the seed + ) + assert ( + len([relation for relation in relations if relation.is_dynamic_table]) + == self.dynamic_tables + ) + + +class TestShowIcebergObjects(ShowObjectsBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": _MODEL_ICEBERG} + + def test_quoting_ignore_flag_doesnt_break_iceberg_metadata(self, project): + """https://github.com/dbt-labs/dbt-snowflake/issues/1227 + + The list relations function involves a metadata sub-query. Regardless of + QUOTED_IDENTIFIERS_IGNORE_CASE, this function will fail without proper + normalization within the encapsulating python function after the macro invocation + returns. This test verifies that normalization is working. + """ + run_dbt(["run"]) + + self.list_relations_without_caching(project) diff --git a/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_special_characters.py b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_special_characters.py new file mode 100644 index 000000000..4dce56da9 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/list_relations_tests/test_special_characters.py @@ -0,0 +1,25 @@ +import pytest +from dbt.tests.util import run_dbt + + +TABLE_BASE_SQL = """ +-- models/my_model.sql +{{ config(schema = '1_contains_special*character$') }} +select 1 as id +""" + + +class TestSpecialCharactersInSchema: + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"quoting": {"schema": True}} + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": TABLE_BASE_SQL, + } + + def test_schema_with_special_chars(self, project): + run_dbt(["run", "-s", "my_model"]) diff --git a/dbt-snowflake/tests/functional/adapter/python_model_tests/_files.py b/dbt-snowflake/tests/functional/adapter/python_model_tests/_files.py new file mode 100644 index 000000000..dd69f37fa --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/python_model_tests/_files.py @@ -0,0 +1,48 @@ +# __table +TRANSIENT_TRUE_TABLE = """ +import pandas + +def model(dbt, session): + dbt.config(transient=True) + return pandas.DataFrame([[1,2]] * 10, columns=['test', 'test2']) +""" + + +TRANSIENT_FALSE_TABLE = """ +import pandas + +def model(dbt, session): + dbt.config(transient=False) + return pandas.DataFrame([[1,2]] * 10, columns=['test', 'test2']) +""" + + +TRANSIENT_NONE_TABLE = """ +import pandas + +def model(dbt, session): + dbt.config(transient=None) + return pandas.DataFrame([[1,2]] * 10, columns=['test', 'test2']) +""" + + +TRANSIENT_UNSET_TABLE = """ +import pandas + +def model(dbt, session): + return pandas.DataFrame([[1,2]] * 10, columns=['test', 'test2']) +""" + + +MACRO__DESCRIBE_TABLES = """ +{% macro snowflake__test__describe_tables() %} + {%- set _sql -%} + show tables; + select "name", "kind" + from table(result_scan(last_query_id())) + {%- endset %} + {% set _table = run_query(_sql) %} + + {% do return(_table) %} +{% endmacro %} +""" diff --git a/dbt-snowflake/tests/functional/adapter/python_model_tests/test_table_type.py b/dbt-snowflake/tests/functional/adapter/python_model_tests/test_table_type.py new file mode 100644 index 000000000..df1f34ac4 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/python_model_tests/test_table_type.py @@ -0,0 +1,35 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.adapter.python_model_tests import _files + + +class TestTableType: + @pytest.fixture(scope="class") + def macros(self): + return {"snowflake__test__describe_tables.sql": _files.MACRO__DESCRIBE_TABLES} + + @pytest.fixture(scope="class") + def models(self): + return { + # __table + "TRANSIENT_TRUE_TABLE.py": _files.TRANSIENT_TRUE_TABLE, + "TRANSIENT_FALSE_TABLE.py": _files.TRANSIENT_FALSE_TABLE, + "TRANSIENT_NONE_TABLE.py": _files.TRANSIENT_NONE_TABLE, + "TRANSIENT_UNSET_TABLE.py": _files.TRANSIENT_UNSET_TABLE, + } + + def test_expected_table_types_are_created(self, project): + run_dbt(["run"]) + expected_table_types = { + # (name, kind) - TABLE == permanent + ("TRANSIENT_TRUE_TABLE", "TRANSIENT"), + ("TRANSIENT_FALSE_TABLE", "TABLE"), + ("TRANSIENT_NONE_TABLE", "TABLE"), + ("TRANSIENT_UNSET_TABLE", "TRANSIENT"), + } + with project.adapter.connection_named("__test"): + agate_table = project.adapter.execute_macro("snowflake__test__describe_tables") + actual_table_types = {(row.get("name"), row.get("kind")) for row in agate_table.rows} + assert actual_table_types == expected_table_types diff --git a/dbt-snowflake/tests/functional/adapter/query_comment_tests/test_query_comments.py b/dbt-snowflake/tests/functional/adapter/query_comment_tests/test_query_comments.py new file mode 100644 index 000000000..8fbfcbdd8 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/query_comment_tests/test_query_comments.py @@ -0,0 +1,38 @@ +import pytest +from dbt.tests.adapter.query_comment.test_query_comment import ( + BaseQueryComments, + BaseMacroQueryComments, + BaseMacroArgsQueryComments, + BaseMacroInvalidQueryComments, + BaseNullQueryComments, + BaseEmptyQueryComments, +) + + +class TestQueryCommentsSnowflake(BaseQueryComments): + pass + + +class TestMacroQueryCommentsSnowflake(BaseMacroQueryComments): + pass + + +class TestMacroArgsQueryCommentsSnowflake(BaseMacroArgsQueryComments): + @pytest.mark.skip( + "This test is incorrectly comparing the version of `dbt-core`" + "to the version of `dbt-snowflake`, which is not always the same." + ) + def test_matches_comment(self, project, get_package_version): + pass + + +class TestMacroInvalidQueryCommentsSnowflake(BaseMacroInvalidQueryComments): + pass + + +class TestNullQueryCommentsSnowflake(BaseNullQueryComments): + pass + + +class TestEmptyQueryCommentsSnowflake(BaseEmptyQueryComments): + pass diff --git a/dbt-snowflake/tests/functional/adapter/simple_copy/fixtures.py b/dbt-snowflake/tests/functional/adapter/simple_copy/fixtures.py new file mode 100644 index 000000000..44707e33c --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/simple_copy/fixtures.py @@ -0,0 +1,759 @@ +_SEEDS__SEED_UPDATE = """ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 +102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 +103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 +104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 +105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 +106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 +107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 +108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 +109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 +110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 +111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 +112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 +113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 +114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 +115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 +116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 +117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 +118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 +119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 +120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 +121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 +122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 +123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 +124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 +125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 +126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 +127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 +128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 +129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 +130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 +131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 +132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 +133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 +134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 +135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 +136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 +137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 +138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 +139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 +140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 +141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 +142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 +143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 +144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 +145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 +146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 +147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 +148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 +149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 +150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 +151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 +152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 +153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 +154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 +155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 +156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 +157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 +158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 +159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 +160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 +161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 +162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 +163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 +164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 +165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 +166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 +167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 +168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 +169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 +170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 +171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 +172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 +173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 +174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 +175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 +176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 +177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 +178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 +179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 +180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 +181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 +182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 +183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 +184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 +185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 +186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 +187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 +188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 +189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 +190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 +191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 +192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 +193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 +194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 +195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 +196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 +197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 +198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 +199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 +200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 +""" + +_SEEDS__SEED_MERGE_INITIAL = """ load_date,id,first_name,last_name,email,gender,ip_address +2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +2021-03-05,20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +2021-03-05,92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +""" + + +_SEEDS__SEED_MERGE_UPDATE = """load_date,id,first_name,last_name,email,gender,ip_address +2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +2021-03-05,20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +2021-03-05,92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +2021-03-06,20,Andrew,Davis,adavisj@reddit.com,Male,9.255.67.25 +2021-03-06,83,Josh,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +2021-03-06,92,Angela,Scott,ascott2j@goodreads.com,Female,98.119.208.155 +2021-03-06,101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 +2021-03-06,102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 +2021-03-06,103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 +2021-03-06,104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 +2021-03-06,105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 +2021-03-06,106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 +2021-03-06,107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 +2021-03-06,108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 +2021-03-06,109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 +2021-03-06,110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 +2021-03-06,111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 +2021-03-06,112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 +2021-03-06,113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 +2021-03-06,114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 +2021-03-06,115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 +2021-03-06,116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 +2021-03-06,117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 +2021-03-06,118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 +2021-03-06,119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 +2021-03-06,120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 +2021-03-06,121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 +2021-03-06,122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 +2021-03-06,123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 +2021-03-06,124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 +2021-03-06,125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 +2021-03-06,126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 +2021-03-06,127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 +2021-03-06,128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 +2021-03-06,129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 +2021-03-06,130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 +2021-03-06,131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 +2021-03-06,132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 +2021-03-06,133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 +2021-03-06,134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 +2021-03-06,135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 +2021-03-06,136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 +2021-03-06,137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 +2021-03-06,138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 +2021-03-06,139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 +2021-03-06,140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 +2021-03-06,141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 +2021-03-06,142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 +2021-03-06,143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 +2021-03-06,144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 +2021-03-06,145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 +2021-03-06,146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 +2021-03-06,147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 +2021-03-06,148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 +2021-03-06,149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 +2021-03-06,150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 +2021-03-06,151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 +2021-03-06,152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 +2021-03-06,153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 +2021-03-06,154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 +2021-03-06,155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 +2021-03-06,156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 +2021-03-06,157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 +2021-03-06,158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 +2021-03-06,159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 +2021-03-06,160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 +2021-03-06,161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 +2021-03-06,162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 +2021-03-06,163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 +2021-03-06,164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 +2021-03-06,165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 +2021-03-06,166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 +2021-03-06,167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 +2021-03-06,168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 +2021-03-06,169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 +2021-03-06,170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 +2021-03-06,171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 +2021-03-06,172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 +2021-03-06,173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 +2021-03-06,174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 +2021-03-06,175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 +2021-03-06,176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 +2021-03-06,177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 +2021-03-06,178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 +2021-03-06,179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 +2021-03-06,180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 +2021-03-06,181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 +2021-03-06,182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 +2021-03-06,183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 +2021-03-06,184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 +2021-03-06,185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 +2021-03-06,186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 +2021-03-06,187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 +2021-03-06,188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 +2021-03-06,189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 +2021-03-06,190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 +2021-03-06,191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 +2021-03-06,192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 +2021-03-06,193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 +2021-03-06,194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 +2021-03-06,195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 +2021-03-06,196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 +2021-03-06,197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 +2021-03-06,198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 +2021-03-06,199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 +2021-03-06,200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 +""" + +_SEEDS__SEED_MERGE_EXPECTED = """load_date,id,first_name,last_name,email,gender,ip_address +2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +2021-03-05,20,Andrew,Davis,adavisj@reddit.com,Male,9.255.67.25 +2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +2021-03-05,92,Angela,Rogers,ascott2j@goodreads.com,Female,98.119.208.155 +2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +2021-03-06,101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 +2021-03-06,102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 +2021-03-06,103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 +2021-03-06,104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 +2021-03-06,105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 +2021-03-06,106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 +2021-03-06,107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 +2021-03-06,108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 +2021-03-06,109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 +2021-03-06,110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 +2021-03-06,111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 +2021-03-06,112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 +2021-03-06,113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 +2021-03-06,114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 +2021-03-06,115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 +2021-03-06,116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 +2021-03-06,117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 +2021-03-06,118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 +2021-03-06,119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 +2021-03-06,120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 +2021-03-06,121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 +2021-03-06,122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 +2021-03-06,123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 +2021-03-06,124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 +2021-03-06,125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 +2021-03-06,126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 +2021-03-06,127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 +2021-03-06,128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 +2021-03-06,129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 +2021-03-06,130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 +2021-03-06,131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 +2021-03-06,132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 +2021-03-06,133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 +2021-03-06,134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 +2021-03-06,135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 +2021-03-06,136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 +2021-03-06,137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 +2021-03-06,138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 +2021-03-06,139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 +2021-03-06,140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 +2021-03-06,141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 +2021-03-06,142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 +2021-03-06,143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 +2021-03-06,144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 +2021-03-06,145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 +2021-03-06,146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 +2021-03-06,147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 +2021-03-06,148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 +2021-03-06,149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 +2021-03-06,150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 +2021-03-06,151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 +2021-03-06,152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 +2021-03-06,153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 +2021-03-06,154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 +2021-03-06,155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 +2021-03-06,156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 +2021-03-06,157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 +2021-03-06,158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 +2021-03-06,159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 +2021-03-06,160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 +2021-03-06,161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 +2021-03-06,162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 +2021-03-06,163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 +2021-03-06,164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 +2021-03-06,165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 +2021-03-06,166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 +2021-03-06,167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 +2021-03-06,168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 +2021-03-06,169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 +2021-03-06,170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 +2021-03-06,171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 +2021-03-06,172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 +2021-03-06,173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 +2021-03-06,174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 +2021-03-06,175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 +2021-03-06,176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 +2021-03-06,177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 +2021-03-06,178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 +2021-03-06,179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 +2021-03-06,180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 +2021-03-06,181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 +2021-03-06,182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 +2021-03-06,183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 +2021-03-06,184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 +2021-03-06,185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 +2021-03-06,186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 +2021-03-06,187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 +2021-03-06,188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 +2021-03-06,189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 +2021-03-06,190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 +2021-03-06,191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 +2021-03-06,192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 +2021-03-06,193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 +2021-03-06,194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 +2021-03-06,195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 +2021-03-06,196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 +2021-03-06,197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 +2021-03-06,198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 +2021-03-06,199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 +2021-03-06,200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 +""" + + +_MODELS__INCREMENTAL_OVERWRITE = """ +{{ config(materialized='incremental', unique_key='id') }} + +-- this will fail on snowflake with "merge" due +-- to the nondeterministic join on id + +select 1 as id +union all +select 1 as id +""" + +_MODELS__INCREMENTAL_UPDATE_COLS = """ +{{ + config( + materialized = "incremental", + unique_key = "id", + merge_update_columns = ["email", "ip_address"] + ) +}} + + +select * +from {{ ref('seed') }} + +{% if is_incremental() %} + + where load_date > (select max(load_date) from {{this}}) + +{% endif %} +""" + + +_TESTS__GET_RELATION_QUOTING = """ {%- set tgt = ref('seed') -%} +{%- set got = adapter.get_relation(database=tgt.database, schema=tgt.schema, identifier=tgt.identifier) | string -%} +{% set replaced = got.replace('"', '-') %} +{% set expected = "-" + tgt.database.upper() + '-.-' + tgt.schema.upper() + '-.-' + tgt.identifier.upper() + '-' %} + +with cte as ( + select '{{ replaced }}' as name +) +select * from cte where name not like '{{ expected }}' +""" diff --git a/dbt-snowflake/tests/functional/adapter/simple_copy/test_simple_copy.py b/dbt-snowflake/tests/functional/adapter/simple_copy/test_simple_copy.py new file mode 100644 index 000000000..4d6cb3cb1 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/simple_copy/test_simple_copy.py @@ -0,0 +1,145 @@ +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, rm_file, write_file, check_relations_equal + +from dbt.tests.adapter.simple_copy.test_simple_copy import ( + SimpleCopySetup, + SimpleCopyBase, + EmptyModelsArentRunBase, +) + +from tests.functional.adapter.simple_copy.fixtures import ( + _MODELS__INCREMENTAL_OVERWRITE, + _MODELS__INCREMENTAL_UPDATE_COLS, + _SEEDS__SEED_MERGE_EXPECTED, + _SEEDS__SEED_MERGE_INITIAL, + _SEEDS__SEED_MERGE_UPDATE, + _SEEDS__SEED_UPDATE, + _TESTS__GET_RELATION_QUOTING, +) + + +class TestSimpleCopyBase(SimpleCopyBase): + @pytest.fixture(scope="class") + def tests(self): + return {"get_relation_test.sql": _TESTS__GET_RELATION_QUOTING} + + def test_simple_copy(self, project): + super().test_simple_copy(project) + run_dbt(["test"]) + + +class TestSimpleCopyBaseQuotingOff(SimpleCopyBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"quoting": {"identifier": False}} + + @pytest.fixture(scope="class") + def tests(self): + return {"get_relation_test.sql": _TESTS__GET_RELATION_QUOTING} + + def test_simple_copy_quoting_off(self, project): + super().test_simple_copy(project) + run_dbt(["test"]) + + @pytest.mark.skip(reason="Already run and test case above; no need to run again") + def test_simple_copy(self): + pass + + @pytest.mark.skip(reason="Already run and test case above; no need to run again") + def test_simple_copy_with_materialized_views(self): + pass + + +class TestEmptyModelsArentRun(EmptyModelsArentRunBase): + pass + + +file_append = """ +quoting: + identifier: true +""" + + +class TestSimpleCopyBaseQuotingSwitch(SimpleCopySetup): + @pytest.fixture(scope="class") + def tests(self): + return {"get_relation_test.sql": _TESTS__GET_RELATION_QUOTING} + + def test_seed_quoting_switch(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Update seed file + main_seed_file = project.project_root / Path("seeds") / Path("seed.csv") + rm_file(main_seed_file) + write_file(_SEEDS__SEED_UPDATE, main_seed_file) + + # Change the profile temporarily + dbt_project_yml = project.project_root / Path("dbt_project.yml") + with open(dbt_project_yml, "r+") as f: + dbt_project_yml_contents = f.read() + f.write(file_append) + run_dbt(["seed"], expect_pass=False) + + with open(dbt_project_yml, "w") as f: + f.write(dbt_project_yml_contents) + run_dbt(["test"]) + + +inc_strat_yml = """ +models: + incremental_strategy: "delete+insert" +""" + + +class TestSnowflakeIncrementalOverwrite(SimpleCopySetup): + @pytest.fixture(scope="class") + def models(self): + return {"incremental_overwrite.sql": _MODELS__INCREMENTAL_OVERWRITE} + + def test__snowflake__incremental_overwrite(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 1 + + # Fails using 'merge' strategy because there's a duplicate 'id' + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 1 + + # Setting the incremental_strategy should make this succeed + dbt_project_yml = project.project_root / Path("dbt_project.yml") + with open(dbt_project_yml, "a") as f: + f.write(inc_strat_yml) + + results = run_dbt(["run"]) + assert len(results) == 1 + + +class TestIncrementalMergeColumns(SimpleCopySetup): + @pytest.fixture(scope="class") + def models(self): + return {"incremental_update_cols.sql": _MODELS__INCREMENTAL_UPDATE_COLS} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEEDS__SEED_MERGE_INITIAL, + "expected_result.csv": _SEEDS__SEED_MERGE_EXPECTED, + } + + def seed_and_run(self): + run_dbt(["seed"]) + run_dbt(["run"]) + + def test__snowflake__incremental_merge_columns(self, project): + self.seed_and_run() + + main_seed_file = project.project_root / Path("seeds") / Path("seed.csv") + rm_file(main_seed_file) + write_file(_SEEDS__SEED_MERGE_UPDATE, main_seed_file) + self.seed_and_run() + check_relations_equal(project.adapter, ["incremental_update_cols", "expected_result"]) diff --git a/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed.py b/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed.py new file mode 100644 index 000000000..af28b394d --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed.py @@ -0,0 +1,16 @@ +import pytest + +from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase +from dbt.tests.util import run_dbt + + +class TestSimpleBigSeedBatched(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + seed_data = ["seed_id"] + seed_data.extend([str(i) for i in range(20_000)]) + return {"big_batched_seed.csv": "\n".join(seed_data)} + + def test_big_batched_seed(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 1 diff --git a/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed_override.py b/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed_override.py new file mode 100644 index 000000000..4dc559854 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/simple_seed/test_simple_seed_override.py @@ -0,0 +1,84 @@ +import pytest +from dbt.tests.adapter.simple_seed.test_seed_type_override import BaseSimpleSeedColumnOverride +from dbt.tests.adapter.utils.base_utils import run_dbt + +_SCHEMA_YML = """ +version: 2 +seeds: +- name: seed_enabled + columns: + - name: birthday + data_tests: + - column_type: + type: character varying(16777216) + - name: seed_id + data_tests: + - column_type: + type: FLOAT + +- name: seed_tricky + columns: + - name: seed_id + data_tests: + - column_type: + type: NUMBER(38,0) + - name: seed_id_str + data_tests: + - column_type: + type: character varying(16777216) + - name: a_bool + data_tests: + - column_type: + type: BOOLEAN + - name: looks_like_a_bool + data_tests: + - column_type: + type: character varying(16777216) + - name: a_date + data_tests: + - column_type: + type: TIMESTAMP_NTZ + - name: looks_like_a_date + data_tests: + - column_type: + type: character varying(16777216) + - name: relative + data_tests: + - column_type: + type: character varying(16777216) + - name: weekday + data_tests: + - column_type: + type: character varying(16777216) +""".lstrip() + + +class TestSimpleSeedColumnOverride(BaseSimpleSeedColumnOverride): + @pytest.fixture(scope="class") + def schema(self): + return "simple_seed" + + @pytest.fixture(scope="class") + def models(self): + return {"models-snowflake.yml": _SCHEMA_YML} + + @staticmethod + def seed_enabled_types(): + return { + "seed_id": "FLOAT", + "birthday": "TEXT", + } + + @staticmethod + def seed_tricky_types(): + return { + "seed_id_str": "TEXT", + "looks_like_a_bool": "TEXT", + "looks_like_a_date": "TEXT", + } + + def test_snowflake_simple_seed_with_column_override_snowflake(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 2 + test_results = run_dbt(["test"]) + assert len(test_results) == 10 diff --git a/dbt-snowflake/tests/functional/adapter/statement_test/seeds.py b/dbt-snowflake/tests/functional/adapter/statement_test/seeds.py new file mode 100644 index 000000000..5f29e9ca4 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/statement_test/seeds.py @@ -0,0 +1,109 @@ +seeds_csv = """ +ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +""".lstrip() + +statement_expected_csv = """ +SOURCE,VALUE +matrix,100 +table,100 +""".lstrip() diff --git a/dbt-snowflake/tests/functional/adapter/statement_test/test_statements.py b/dbt-snowflake/tests/functional/adapter/statement_test/test_statements.py new file mode 100644 index 000000000..417870c68 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/statement_test/test_statements.py @@ -0,0 +1,53 @@ +import pytest +from dbt.tests.util import check_relations_equal, run_dbt +from tests.functional.adapter.statement_test.seeds import seeds_csv, statement_expected_csv + +_STATEMENT_ACTUAL_SQL = """ +-- {{ ref('seed') }} + +{%- call statement('test_statement', fetch_result=True) -%} + + select + count(*) as "num_records" + + from {{ ref('seed') }} + +{%- endcall -%} + +{% set result = load_result('test_statement') %} + +{% set res_table = result['table'] %} +{% set res_matrix = result['data'] %} + +{% set matrix_value = res_matrix[0][0] %} +{% set table_value = res_table[0]['num_records'] %} + +select 'matrix' as source, {{ matrix_value }} as value +union all +select 'table' as source, {{ table_value }} as value +""".lstrip() + + +class TestStatements: + @pytest.fixture(scope="class") + def models(self): + return {"statement_actual.sql": _STATEMENT_ACTUAL_SQL} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seeds_csv, + "statement_expected.csv": statement_expected_csv, + } + + def test_snowflake_statements(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 2 + results = run_dbt() + assert len(results) == 1 + + db_with_schema = f"{project.database}.{project.test_schema}" + check_relations_equal( + project.adapter, + [f"{db_with_schema}.STATEMENT_ACTUAL", f"{db_with_schema}.STATEMENT_EXPECTED"], + ) diff --git a/dbt-snowflake/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py b/dbt-snowflake/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py new file mode 100644 index 000000000..b3bce12c3 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py @@ -0,0 +1,32 @@ +from dbt.tests.adapter.store_test_failures_tests import basic +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + TestStoreTestFailures, +) + + +class TestSnowflakeStoreTestFailures(TestStoreTestFailures): + pass + + +class TestStoreTestFailuresAsInteractions(basic.StoreTestFailuresAsInteractions): + pass + + +class TestStoreTestFailuresAsProjectLevelOff(basic.StoreTestFailuresAsProjectLevelOff): + pass + + +class TestStoreTestFailuresAsProjectLevelView(basic.StoreTestFailuresAsProjectLevelView): + pass + + +class TestStoreTestFailuresAsGeneric(basic.StoreTestFailuresAsGeneric): + pass + + +class TestStoreTestFailuresAsProjectLevelEphemeral(basic.StoreTestFailuresAsProjectLevelEphemeral): + pass + + +class TestStoreTestFailuresAsExceptions(basic.StoreTestFailuresAsExceptions): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_aliases.py b/dbt-snowflake/tests/functional/adapter/test_aliases.py new file mode 100644 index 000000000..d038a37e9 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_aliases.py @@ -0,0 +1,29 @@ +import pytest +from dbt.tests.adapter.aliases.test_aliases import BaseAliases + +MACROS__SNOWFLAKE_CAST_SQL = """ +{% macro snowflake__string_literal(s) %} + cast('{{ s }}' as string) +{% endmacro %} +""" + +MACROS__EXPECT_VALUE_SQL = """ +-- cross-db compatible test, similar to accepted_values + +{% test expect_value(model, field, value) %} + +select * +from {{ model }} +where {{ field }} != '{{ value }}' + +{% endtest %} +""" + + +class TestAliasesSnowflake(BaseAliases): + @pytest.fixture(scope="class") + def macros(self): + return { + "snowflake_cast.sql": MACROS__SNOWFLAKE_CAST_SQL, + "expect_value.sql": MACROS__EXPECT_VALUE_SQL, + } diff --git a/dbt-snowflake/tests/functional/adapter/test_anonymous_usage_stats.py b/dbt-snowflake/tests/functional/adapter/test_anonymous_usage_stats.py new file mode 100644 index 000000000..bc5a4d334 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_anonymous_usage_stats.py @@ -0,0 +1,43 @@ +from dbt.tests.util import run_dbt_and_capture +import pytest + + +ANONYMOUS_USAGE_MESSAGE = """ +sys._xoptions['snowflake_partner_attribution'].append("dbtLabs_dbtPython") +""".strip() + + +MY_PYTHON_MODEL = """ +import pandas + +def model(dbt, session): + dbt.config(materialized='table') + data = [[1,2]] * 10 + return pandas.DataFrame(data, columns=['test', 'test2']) +""" + + +class AnonymousUsageStatsBase: + @pytest.fixture(scope="class") + def models(self): + return {"my_python_model.py": MY_PYTHON_MODEL} + + +class TestAnonymousUsageStatsOn(AnonymousUsageStatsBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"send_anonymous_usage_stats": True}} + + def test_stats_get_sent(self, project): + _, logs = run_dbt_and_capture(["--debug", "run"]) + assert ANONYMOUS_USAGE_MESSAGE in logs + + +class TestAnonymousUsageStatsOff(AnonymousUsageStatsBase): + @pytest.fixture(scope="class") + def project_config_update(self, dbt_profile_target): + return {"flags": {"send_anonymous_usage_stats": False}} + + def test_stats_do_not_get_sent(self, project): + _, logs = run_dbt_and_capture(["--debug", "run"]) + assert ANONYMOUS_USAGE_MESSAGE not in logs diff --git a/dbt-snowflake/tests/functional/adapter/test_basic.py b/dbt-snowflake/tests/functional/adapter/test_basic.py new file mode 100644 index 000000000..1a79f672b --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_basic.py @@ -0,0 +1,243 @@ +import pytest + +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( + BaseSingularTestsEphemeral, +) +from dbt.tests.adapter.basic.test_get_catalog_for_single_relation import ( + BaseGetCatalogForSingleRelation, +) +from dbt.tests.adapter.basic.test_empty import BaseEmpty +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog, no_stats +from dbt_common.contracts.metadata import CatalogTable, TableMetadata, ColumnMetadata, StatsItem + +from dbt.adapters.snowflake.relation_configs import SnowflakeRelationType +from tests.functional.adapter.expected_stats import snowflake_stats + + +class TestSimpleMaterializationsSnowflake(BaseSimpleMaterializations): + pass + + +class TestSingularTestsSnowflake(BaseSingularTests): + pass + + +class TestGetCatalogForSingleRelationSnowflake(BaseGetCatalogForSingleRelation): + @pytest.fixture(scope="class") + def current_role(self, project): + return project.run_sql("select current_role()", fetch="one")[0] + + @pytest.fixture(scope="class") + def expected_catalog_my_seed(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.Table.upper(), + schema=project.test_schema.upper(), + name="MY_SEED", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=1, + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=2048, + include=True, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_view_model(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.View.upper(), + schema=project.test_schema.upper(), + name="MY_VIEW_MODEL", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=0, + include=False, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=0, + include=False, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_table_model(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.Table.upper(), + schema=project.test_schema.upper(), + name="MY_TABLE_MODEL", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=1, + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=2048, + include=True, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, + ) + + +class TestSingularTestsEphemeralSnowflake(BaseSingularTestsEphemeral): + pass + + +class TestEmptySnowflake(BaseEmpty): + pass + + +class TestEphemeralSnowflake(BaseEphemeral): + pass + + +class TestIncrementalSnowflake(BaseIncremental): + pass + + +class TestGenericTestsSnowflake(BaseGenericTests): + pass + + +class TestSnapshotCheckColsSnowflake(BaseSnapshotCheckCols): + pass + + +class TestSnapshotTimestampSnowflake(BaseSnapshotTimestamp): + pass + + +class TestBaseAdapterMethodSnowflake(BaseAdapterMethod): + @pytest.fixture(scope="class") + def equal_tables(self): + return ["MODEL", "EXPECTED"] + + +class TestDocsGenerateSnowflake(BaseDocsGenerate): + @pytest.fixture(scope="class") + def get_role(self, project): + return project.run_sql("select current_role()", fetch="one")[0] + + @pytest.fixture(scope="class") + def expected_catalog(self, project, get_role): + return base_expected_catalog( + project, + role=get_role, + id_type="NUMBER", + text_type="TEXT", + time_type="TIMESTAMP_NTZ", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + seed_stats=snowflake_stats(), + case=lambda x: x.upper(), + case_columns=False, + ) diff --git a/dbt-snowflake/tests/functional/adapter/test_caching.py b/dbt-snowflake/tests/functional/adapter/test_caching.py new file mode 100644 index 000000000..f5998c3b2 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_caching.py @@ -0,0 +1,17 @@ +from dbt.tests.adapter.caching.test_caching import ( + BaseCachingLowercaseModel, + BaseCachingUppercaseModel, + BaseCachingSelectedSchemaOnly, +) + + +class TestCachingLowerCaseModel(BaseCachingLowercaseModel): + pass + + +class TestCachingUppercaseModel(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnly(BaseCachingSelectedSchemaOnly): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_changing_relation_type.py b/dbt-snowflake/tests/functional/adapter/test_changing_relation_type.py new file mode 100644 index 000000000..404791b79 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_changing_relation_type.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator + + +class TestSnowflakeChangeRelationTypes(BaseChangeRelationTypeValidator): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_concurrency.py b/dbt-snowflake/tests/functional/adapter/test_concurrency.py new file mode 100644 index 000000000..0efbb55ce --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_concurrency.py @@ -0,0 +1,22 @@ +from dbt.tests.util import run_dbt, check_relations_equal, rm_file, write_file +from dbt.tests.adapter.concurrency.test_concurrency import BaseConcurrency, seeds__update_csv + + +class TestConncurenncySnowflake(BaseConcurrency): + def test_conncurrency_snowflake(self, project): + run_dbt(["seed", "--select", "seed"]) + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"]) + check_relations_equal(project.adapter, ["SEED", "DEP"]) + check_relations_equal(project.adapter, ["SEED", "TABLE_A"]) + check_relations_equal(project.adapter, ["SEED", "TABLE_B"]) + + rm_file(project.project_root, "seeds", "seed.csv") + write_file(seeds__update_csv, project.project_root + "/seeds", "seed.csv") + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"]) + check_relations_equal(project.adapter, ["SEED", "DEP"]) + check_relations_equal(project.adapter, ["SEED", "TABLE_A"]) + check_relations_equal(project.adapter, ["SEED", "TABLE_B"]) diff --git a/dbt-snowflake/tests/functional/adapter/test_constraints.py b/dbt-snowflake/tests/functional/adapter/test_constraints.py new file mode 100644 index 000000000..03adc3bed --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_constraints.py @@ -0,0 +1,202 @@ +import pytest + +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseTableContractSqlHeader, + BaseIncrementalContractSqlHeader, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRollback, + BaseModelConstraintsRuntimeEnforcement, + BaseConstraintQuotedColumn, +) + +from dbt.tests.adapter.constraints.fixtures import ( + model_contract_header_schema_yml, +) + +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} +{% call set_sql_header(config) %} +SET MY_VARIABLE='test'; +{% endcall %} +SELECT $MY_VARIABLE as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} +{% call set_sql_header(config) %} +SET MY_VARIABLE='test'; +{% endcall %} +SELECT $MY_VARIABLE as column_name +""" + +_expected_sql_snowflake = """ +create or replace transient table ( + id integer not null primary key references (id) unique, + color text, + date_day text +) as ( select + id, + color, + date_day from + ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq +); +""" + + +class SnowflakeColumnEqualSetup: + @pytest.fixture + def int_type(self): + return "FIXED" + + @pytest.fixture + def schema_int_type(self): + return "INT" + + @pytest.fixture + def data_types(self, int_type, schema_int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "DATE"], + ["true", "boolean", "BOOLEAN"], + ["'2013-11-03 00:00:00-07'::timestamptz", "timestamp_tz", "TIMESTAMP_TZ"], + ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "TIMESTAMP_NTZ"], + ["ARRAY_CONSTRUCT('a','b','c')", "array", "ARRAY"], + ["ARRAY_CONSTRUCT(1,2,3)", "array", "ARRAY"], + ["TO_GEOGRAPHY('POINT(-122.35 37.55)')", "geography", "GEOGRAPHY"], + ["TO_GEOMETRY('POINT(1820.12 890.56)')", "geometry", "GEOMETRY"], + [ + """TO_VARIANT(PARSE_JSON('{"key3": "value3", "key4": "value4"}'))""", + "variant", + "VARIANT", + ], + ] + + +class TestSnowflakeTableConstraintsColumnsEqual( + SnowflakeColumnEqualSetup, BaseTableConstraintsColumnsEqual +): + pass + + +class TestSnowflakeViewConstraintsColumnsEqual( + SnowflakeColumnEqualSetup, BaseViewConstraintsColumnsEqual +): + pass + + +class TestSnowflakeIncrementalConstraintsColumnsEqual( + SnowflakeColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual +): + pass + + +class TestSnowflakeTableContractsSqlHeader(BaseTableContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestSnowflakeIncrementalContractsSqlHeader(BaseIncrementalContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestSnowflakeTableConstraintsDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_snowflake + + +class TestSnowflakeIncrementalConstraintsDdlEnforcement( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_snowflake + + +class TestSnowflakeTableConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NULL result in a non-nullable column"] + + +class TestSnowflakeIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NULL result in a non-nullable column"] + + +class TestSnowflakeModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create or replace transient table ( + id integer not null, + color text, + date_day text, + primary key (id), + constraint strange_uniqueness_requirement unique (color, date_day), + foreign key (id) references (id) +) as ( select + id, + color, + date_day from + ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq +); +""" + + +class TestSnowflakeConstraintQuotedColumn(BaseConstraintQuotedColumn): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create or replace transient table ( + id integer not null, + "from" text not null, + date_day text +) as ( + select id, "from", date_day + from ( + select + 'blue' as "from", + 1 as id, + '2019-01-01' as date_day + ) as model_subq +); +""" diff --git a/dbt-snowflake/tests/functional/adapter/test_ephemeral.py b/dbt-snowflake/tests/functional/adapter/test_ephemeral.py new file mode 100644 index 000000000..ff8cee913 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_ephemeral.py @@ -0,0 +1,12 @@ +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti +from dbt.tests.util import run_dbt, check_relations_equal + + +class TestEphemeralMultiSnowflake(BaseEphemeralMulti): + def test_ephemeral_multi(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + check_relations_equal( + project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"] + ) diff --git a/dbt-snowflake/tests/functional/adapter/test_get_last_relation_modified.py b/dbt-snowflake/tests/functional/adapter/test_get_last_relation_modified.py new file mode 100644 index 000000000..56e8d46bf --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_get_last_relation_modified.py @@ -0,0 +1,152 @@ +import os +import pytest +from unittest import mock + +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.capability import Capability, CapabilityDict +from dbt.cli.main import dbtRunner + + +freshness_via_metadata_schema_yml = """ +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_table +""" + +freshness_metadata_schema_batch_yml = """ +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_table + - name: test_table2 + - name: test_table_with_loaded_at_field + loaded_at_field: my_loaded_at_field +""" + + +class SetupGetLastRelationModified: + @pytest.fixture(scope="class", autouse=True) + def set_env_vars(self, project): + os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema + yield + del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + + @pytest.fixture(scope="class") + def custom_schema(self, project, set_env_vars): + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + ) + project.adapter.drop_schema(relation) + project.adapter.create_schema(relation) + + yield relation.schema + + with project.adapter.connection_named("__test"): + project.adapter.drop_schema(relation) + + +class TestGetLastRelationModified(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": freshness_via_metadata_schema_yml} + + def test_get_last_relation_modified(self, project, set_env_vars, custom_schema): + project.run_sql( + f"create table {custom_schema}.test_table (id integer autoincrement, name varchar(100) not null);" + ) + + warning_or_error = False + + def probe(e): + nonlocal warning_or_error + if e.info.level in ["warning", "error"]: + warning_or_error = True + + runner = dbtRunner(callbacks=[probe]) + runner.invoke(["source", "freshness"]) + + # The 'source freshness' command should succeed without warnings or errors. + assert not warning_or_error + + +class TestGetLastRelationModifiedBatch(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": freshness_metadata_schema_batch_yml} + + def get_freshness_result_for_table(self, table_name, results): + for result in results: + if result.node.name == table_name: + return result + return None + + def test_get_last_relation_modified_batch(self, project, set_env_vars, custom_schema): + project.run_sql( + f"create table {custom_schema}.test_table (id integer autoincrement, name varchar(100) not null);" + ) + project.run_sql( + f"create table {custom_schema}.test_table2 (id integer autoincrement, name varchar(100) not null);" + ) + project.run_sql( + f"create table {custom_schema}.test_table_with_loaded_at_field as (select 1 as id, timestamp '2009-09-15 10:59:43' as my_loaded_at_field);" + ) + + runner = dbtRunner() + freshness_results_batch = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results_batch) == 3 + test_table_batch_result = self.get_freshness_result_for_table( + "test_table", freshness_results_batch + ) + test_table2_batch_result = self.get_freshness_result_for_table( + "test_table2", freshness_results_batch + ) + test_table_with_loaded_at_field_batch_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results_batch + ) + + # Remove TableLastModifiedMetadataBatch and run freshness on same input without batch strategy + capabilities_no_batch = CapabilityDict( + { + capability: support + for capability, support in SnowflakeAdapter.capabilities().items() + if capability != Capability.TableLastModifiedMetadataBatch + } + ) + with mock.patch.object( + SnowflakeAdapter, "capabilities", return_value=capabilities_no_batch + ): + freshness_results = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results) == 3 + test_table_result = self.get_freshness_result_for_table("test_table", freshness_results) + test_table2_result = self.get_freshness_result_for_table("test_table2", freshness_results) + test_table_with_loaded_at_field_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results + ) + + # assert results between batch vs non-batch freshness strategy are equivalent + assert test_table_result.status == test_table_batch_result.status + assert test_table_result.max_loaded_at == test_table_batch_result.max_loaded_at + + assert test_table2_result.status == test_table2_batch_result.status + assert test_table2_result.max_loaded_at == test_table2_batch_result.max_loaded_at + + assert ( + test_table_with_loaded_at_field_batch_result.status + == test_table_with_loaded_at_field_result.status + ) + assert ( + test_table_with_loaded_at_field_batch_result.max_loaded_at + == test_table_with_loaded_at_field_result.max_loaded_at + ) diff --git a/dbt-snowflake/tests/functional/adapter/test_grants.py b/dbt-snowflake/tests/functional/adapter/test_grants.py new file mode 100644 index 000000000..30e687f59 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_grants.py @@ -0,0 +1,64 @@ +import pytest +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +class BaseCopyGrantsSnowflake: + # Try every test case without copy_grants enabled (default), + # and with copy_grants enabled (this base class) + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+copy_grants": True, + }, + "seeds": { + "+copy_grants": True, + }, + "snapshots": { + "+copy_grants": True, + }, + } + + +class TestInvalidGrantsSnowflake(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "does not exist or not authorized" + + def privilege_does_not_exist_error(self): + return "unexpected" + + +class TestModelGrantsSnowflake(BaseModelGrants): + pass + + +class TestModelGrantsCopyGrantsSnowflake(BaseCopyGrantsSnowflake, BaseModelGrants): + pass + + +class TestIncrementalGrantsSnowflake(BaseIncrementalGrants): + pass + + +class TestIncrementalCopyGrantsSnowflake(BaseCopyGrantsSnowflake, BaseIncrementalGrants): + pass + + +class TestSeedGrantsSnowflake(BaseSeedGrants): + pass + + +class TestSeedCopyGrantsSnowflake(BaseCopyGrantsSnowflake, BaseSeedGrants): + pass + + +class TestSnapshotGrants(BaseSnapshotGrants): + pass + + +class TestSnapshotCopyGrantsSnowflake(BaseCopyGrantsSnowflake, BaseSnapshotGrants): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_incremental_microbatch.py b/dbt-snowflake/tests/functional/adapter/test_incremental_microbatch.py new file mode 100644 index 000000000..f087596e1 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_incremental_microbatch.py @@ -0,0 +1,38 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_microbatch import ( + BaseMicrobatch, +) + +# Create input with UTC timestamps +_input_model_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, to_timestamp_tz('2020-01-01 00:00:00-0') as event_time +union all +select 2 as id, to_timestamp_tz('2020-01-02 00:00:00-0') as event_time +union all +select 3 as id, to_timestamp_tz('2020-01-03 00:00:00-0') as event_time +""" + + +# No requirement for a unique_id for snowflake microbatch! +_microbatch_model_no_unique_id_sql = """ +{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} +select * from {{ ref('input_model') }} +""" + + +class TestSnowflakeMicrobatch(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return _microbatch_model_no_unique_id_sql + + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + return _input_model_sql + + @pytest.fixture(scope="class") + def insert_two_rows_sql(self, project) -> str: + test_schema_relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + return f"insert into {test_schema_relation}.input_model (id, event_time) values (4, '2020-01-04 00:00:00-0'), (5, '2020-01-05 00:00:00-0')" diff --git a/dbt-snowflake/tests/functional/adapter/test_persist_docs.py b/dbt-snowflake/tests/functional/adapter/test_persist_docs.py new file mode 100644 index 000000000..4ad575cad --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_persist_docs.py @@ -0,0 +1,75 @@ +import json +import os + +from dbt.tests.util import run_dbt + +from dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) + + +class TestPersistDocs(BasePersistDocs): + def _assert_common_comments(self, *comments): + for comment in comments: + assert '"with double quotes"' in comment + assert """'''abc123'''""" in comment + assert "\n" in comment + assert ( + "Some [$]lbl[$] labeled [$]lbl[$] and [$][$] unlabeled [$][$] dollar-quoting" + in comment + ) + assert "/* comment */" in comment + if os.name == "nt": + assert "--\r\n" in comment or "--\n" in comment + else: + assert "--\n" in comment + + def _assert_has_table_comments(self, table_node): + table_comment = table_node["metadata"]["comment"] + assert table_comment.startswith("Table model description") + + table_id_comment = table_node["columns"]["ID"]["comment"] + assert table_id_comment.startswith("id Column description") + + table_name_comment = table_node["columns"]["NAME"]["comment"] + assert table_name_comment.startswith("Some stuff here and then a call to") + + self._assert_common_comments(table_comment, table_id_comment, table_name_comment) + + def _assert_has_view_comments( + self, view_node, has_node_comments=True, has_column_comments=True + ): + view_comment = view_node["metadata"]["comment"] + if has_node_comments: + assert view_comment.startswith("View model description") + self._assert_common_comments(view_comment) + else: + assert not view_comment + + view_id_comment = view_node["columns"]["ID"]["comment"] + if has_column_comments: + assert view_id_comment.startswith("id Column description") + self._assert_common_comments(view_id_comment) + else: + assert not view_id_comment + + view_name_comment = view_node["columns"]["NAME"]["comment"] + assert not view_name_comment + + +class TestPersistDocsColumnMissing(BasePersistDocsColumnMissing): + def test_missing_column(self, project): + run_dbt(["docs", "generate"]) + with open("target/catalog.json") as fp: + catalog_data = json.load(fp) + assert "nodes" in catalog_data + + table_node = catalog_data["nodes"]["model.test.missing_column"] + table_id_comment = table_node["columns"]["ID"]["comment"] + assert table_id_comment.startswith("test id column description") + + +class TestPersistDocsCommentOnQuotedColumn(BasePersistDocsCommentOnQuotedColumn): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_python_model.py b/dbt-snowflake/tests/functional/adapter/test_python_model.py new file mode 100644 index 000000000..86ea0a346 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_python_model.py @@ -0,0 +1,231 @@ +import pytest +import uuid +from dbt.tests.util import run_dbt, write_file +from dbt.tests.adapter.python_model.test_python_model import ( + BasePythonModelTests, + BasePythonIncrementalTests, +) + +models__simple_python_model = """ +import pandas + +def model(dbt, session): + dbt.config( + materialized='table', + ) + data = [[1,2]] * 10 + return pandas.DataFrame(data, columns=['test', 'test2']) +""" + +models__simple_python_model_v2 = """ +import pandas + +def model(dbt, session): + dbt.config( + materialized='table', + ) + data = [[1,2]] * 10 + return pandas.DataFrame(data, columns=['test1', 'test3']) +""" + +models__custom_target_model = """ +import pandas + +def model(dbt, session): + dbt.config( + materialized="table", + schema="MY_CUSTOM_SCHEMA", + alias="_TEST_PYTHON_MODEL", + ) + + df = pandas.DataFrame({ + 'City': ['Buenos Aires', 'Brasilia', 'Santiago', 'Bogota', 'Caracas'], + 'Country': ['Argentina', 'Brazil', 'Chile', 'Colombia', 'Venezuela'], + 'Latitude': [-34.58, -15.78, -33.45, 4.60, 10.48], + 'Longitude': [-58.66, -47.91, -70.66, -74.08, -66.86] + }) + + return df +""" + + +class TestPythonModelSnowflake(BasePythonModelTests): + pass + + +class TestIncrementalSnowflake(BasePythonIncrementalTests): + pass + + +class TestIncrementalSnowflakeQuoting(BasePythonModelTests): + # ensure that 'dbt.ref()', 'dbt.this()', and py_write_table() all respect quoting + @pytest.fixture(scope="class") + def project_config_update(self): + return {"quoting": {"identifier": True}} + + +class TestChangingSchemaSnowflake: + @pytest.fixture(scope="class") + def models(self): + return {"simple_python_model.py": models__simple_python_model} + + def test_changing_schema(self, project): + run_dbt(["run"]) + write_file( + models__simple_python_model_v2, + project.project_root + "/models", + "simple_python_model.py", + ) + run_dbt(["run"]) + + +USE_IMPORT_MODEL = """ +import sys +from snowflake.snowpark.types import StructType, FloatType, StringType, StructField + +def model( dbt, session): + + dbt.config( + materialized='table', + imports = ['@dbt_integration_test/iris.csv'], + use_anonymous_sproc = False + ) + schema_for_data_file = StructType([ + StructField("length1", FloatType()), + StructField("width1", FloatType()), + StructField("length2", FloatType()), + StructField("width2", FloatType()), + StructField("variety", StringType()), + ]) + df = session.read.schema(schema_for_data_file).option("field_delimiter", ",").schema(schema_for_data_file).csv("@dbt_integration_test/iris.csv") + return df +""" + + +class TestImportSnowflake: + @pytest.fixture(scope="class") + def models(self): + return {"simple_python_model.py": USE_IMPORT_MODEL} + + @pytest.fixture(scope="class") + def seeds(self): + return {"iris.csv": "1,2,3,4,setosa"} + + def test_import(self, project): + project.run_sql("create or replace STAGE dbt_integration_test") + project.run_sql( + f"PUT file://{project.project_root}/seeds/iris.csv @dbt_integration_test/;" + ) + run_dbt(["run"]) + + +# https://github.com/dbt-labs/dbt-snowflake/issues/393 is notorious for being triggered on some +# environments but not others. As of writing this, we don't know the true root cause. This test may +# not fail on all systems with problems regarding custom schema model configurations. +class TestCustomSchemaWorks: + @pytest.fixture(scope="class") + def models(self): + return {"custom_target_model.py": models__custom_target_model} + + @pytest.fixture(scope="function", autouse=True) + def teardown_method(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_MY_CUSTOM_SCHEMA" + ) + project.adapter.drop_schema(relation) + + def test_custom_target(self, project): + results = run_dbt() + assert results[0].node.schema == f"{project.test_schema}_MY_CUSTOM_SCHEMA" + + +EXTERNAL_ACCESS_INTEGRATION_MODE = """ +import pandas +import snowflake.snowpark as snowpark + +def model(dbt, session: snowpark.Session): + dbt.config( + materialized="table", + external_access_integrations=["test_external_access_integration"], + packages=["httpx==0.26.0"] + ) + import httpx + return session.create_dataframe( + pandas.DataFrame( + [{"result": httpx.get(url="https://www.google.com").status_code}] + ) + ) +""" + + +class TestExternalAccessIntegration: + @pytest.fixture(scope="class") + def models(self): + return {"external_access_integration_python_model.py": EXTERNAL_ACCESS_INTEGRATION_MODE} + + def test_external_access_integration(self, project): + project.run_sql( + "create or replace network rule test_network_rule type = host_port mode = egress value_list= ('www.google.com:443');" + ) + project.run_sql( + "create or replace external access integration test_external_access_integration allowed_network_rules = (test_network_rule) enabled = true;" + ) + run_dbt(["run"]) + + +TEST_RUN_ID = uuid.uuid4().hex +TEST_SECRET = f"test_secret_{TEST_RUN_ID}" +TEST_NETWORK_RULE = f"test_network_rule_{TEST_RUN_ID}" +TEST_EXTERNAL_ACCESS_INTEGRATION = f"test_external_access_integration_{TEST_RUN_ID}" +SECRETS_MODE = f""" +import pandas +import snowflake.snowpark as snowpark + +def model(dbt, session: snowpark.Session): + dbt.config( + materialized="table", + secrets={{"secret_variable_name": "{TEST_SECRET}"}}, + external_access_integrations=["{TEST_EXTERNAL_ACCESS_INTEGRATION}"], + ) + import _snowflake + return session.create_dataframe( + pandas.DataFrame( + [{{"secret_value": _snowflake.get_generic_secret_string('secret_variable_name')}}] + ) + ) +""" + + +class TestSecrets: + @pytest.fixture(scope="class") + def models(self): + return {"secret_python_model.py": SECRETS_MODE} + + @pytest.fixture(scope="class") + def profiles_config_update(self): + return {"retry_all": True, "connect_retries": 3} + + def test_secrets(self, project): + project.run_sql( + f"create or replace secret {TEST_SECRET} type = generic_string secret_string='secret value';" + ) + + project.run_sql( + f"create or replace network rule {TEST_NETWORK_RULE} type = host_port mode = egress value_list= ('www.google.com:443');" + ) + + project.run_sql( + f"create or replace external access integration {TEST_EXTERNAL_ACCESS_INTEGRATION} " + f"allowed_network_rules = ({TEST_NETWORK_RULE}) " + f"allowed_authentication_secrets = ({TEST_SECRET}) enabled = true;" + ) + + run_dbt(["run"]) + + project.run_sql(f"drop secret if exists {TEST_SECRET};") + project.run_sql(f"drop network rule if exists {TEST_NETWORK_RULE};") + project.run_sql( + f"drop external access integration if exists {TEST_EXTERNAL_ACCESS_INTEGRATION};" + ) diff --git a/dbt-snowflake/tests/functional/adapter/test_simple_snapshot.py b/dbt-snowflake/tests/functional/adapter/test_simple_snapshot.py new file mode 100644 index 000000000..4db5b2330 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_simple_snapshot.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot + + +class TestSnapshot(BaseSimpleSnapshot): + pass + + +class TestSnapshotCheck(BaseSnapshotCheck): + pass diff --git a/dbt-snowflake/tests/functional/adapter/test_timestamps.py b/dbt-snowflake/tests/functional/adapter/test_timestamps.py new file mode 100644 index 000000000..ead6fb2d9 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/test_timestamps.py @@ -0,0 +1,30 @@ +import pytest +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps + +_MODEL_CURRENT_TIMESTAMP = """ +SELECT {{current_timestamp()}} as current_timestamp, + {{current_timestamp_in_utc_backcompat()}} as current_timestamp_in_utc_backcompat, + {{current_timestamp_backcompat()}} as current_timestamp_backcompat +""" + + +class TestCurrentTimestampSnowflake(BaseCurrentTimestamps): + @pytest.fixture(scope="class") + def models(self): + return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + + @pytest.fixture(scope="class") + def expected_schema(self): + return { + "CURRENT_TIMESTAMP": "TIMESTAMP_TZ", + "CURRENT_TIMESTAMP_IN_UTC_BACKCOMPAT": "TIMESTAMP_NTZ", + "CURRENT_TIMESTAMP_BACKCOMPAT": "TIMESTAMP_NTZ", + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + select convert_timezone('UTC', current_timestamp()) as current_timestamp, + convert_timezone('UTC', current_timestamp::TIMESTAMP)::TIMESTAMP as current_timestamp_in_utc_backcompat, + current_timestamp::TIMESTAMP as current_timestamp_backcompat + """ diff --git a/dbt-snowflake/tests/functional/adapter/unit_testing/test_unit_testing.py b/dbt-snowflake/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..b97be0ac2 --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,39 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], + ["TO_NUMBER('3', 10, 9)", "3"], + ["3::VARIANT", "3"], + ["TO_GEOMETRY('POINT(1820.12 890.56)')", "POINT(1820.12 890.56)"], + ["TO_GEOGRAPHY('POINT(-122.35 37.55)')", "POINT(-122.35 37.55)"], + [ + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + ], + ["['a','b','c']", "['a','b','c']"], + ["[1,2,3]", "[1, 2, 3]"], + ] + + +class TestSnowflakeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestSnowflakeUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass diff --git a/dbt-snowflake/tests/functional/adapter/utils/test_data_types.py b/dbt-snowflake/tests/functional/adapter/utils/test_data_types.py new file mode 100644 index 000000000..3201afcfb --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/utils/test_data_types.py @@ -0,0 +1,35 @@ +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat +from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean + + +class TestTypeBigInt(BaseTypeBigInt): + pass + + +class TestTypeFloat(BaseTypeFloat): + pass + + +class TestTypeInt(BaseTypeInt): + pass + + +class TestTypeNumeric(BaseTypeNumeric): + pass + + +class TestTypeString(BaseTypeString): + pass + + +class TestTypeTimestamp(BaseTypeTimestamp): + pass + + +class TestTypeBoolean(BaseTypeBoolean): + pass diff --git a/dbt-snowflake/tests/functional/adapter/utils/test_utils.py b/dbt-snowflake/tests/functional/adapter/utils/test_utils.py new file mode 100644 index 000000000..c7ec0ce6f --- /dev/null +++ b/dbt-snowflake/tests/functional/adapter/utils/test_utils.py @@ -0,0 +1,156 @@ +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampAware +from dbt.tests.adapter.utils.test_date import BaseDate +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesBackslash +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral + + +class TestAnyValue(BaseAnyValue): + pass + + +class TestArrayAppend(BaseArrayAppend): + pass + + +class TestArrayConcat(BaseArrayConcat): + pass + + +class TestArrayConstruct(BaseArrayConstruct): + pass + + +class TestBoolOr(BaseBoolOr): + pass + + +class TestCast(BaseCast): + pass + + +class TestCastBoolToText(BaseCastBoolToText): + pass + + +class TestConcat(BaseConcat): + pass + + +# Use either BaseCurrentTimestampAware or BaseCurrentTimestampNaive but not both +class TestCurrentTimestamp(BaseCurrentTimestampAware): + pass + + +class TestDate(BaseDate): + pass + + +class TestDateAdd(BaseDateAdd): + pass + + +class TestDateDiff(BaseDateDiff): + pass + + +class TestDateSpine(BaseDateSpine): + pass + + +class TestDateTrunc(BaseDateTrunc): + pass + + +class TestEscapeSingleQuotesQuote(BaseEscapeSingleQuotesQuote): + pass + + +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesBackslash): + pass + + +class TestExcept(BaseExcept): + pass + + +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBeteween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + +class TestHash(BaseHash): + pass + + +class TestIntersect(BaseIntersect): + pass + + +class TestLastDay(BaseLastDay): + pass + + +class TestLength(BaseLength): + pass + + +class TestListagg(BaseListagg): + pass + + +class TestPosition(BasePosition): + pass + + +class TestReplace(BaseReplace): + pass + + +class TestRight(BaseRight): + pass + + +class TestSafeCast(BaseSafeCast): + pass + + +class TestSplitPart(BaseSplitPart): + pass + + +class TestStringLiteral(BaseStringLiteral): + pass diff --git a/dbt-snowflake/tests/functional/auth_tests/test_database_role.py b/dbt-snowflake/tests/functional/auth_tests/test_database_role.py new file mode 100644 index 000000000..c0f93d7d6 --- /dev/null +++ b/dbt-snowflake/tests/functional/auth_tests/test_database_role.py @@ -0,0 +1,68 @@ +import os + +import pytest + +from dbt.tests.util import run_dbt + + +SEED = """ +id +1 +""".strip() + + +MODEL = """ +{{ config( + materialized='incremental', +) }} +select * from {{ ref('my_seed') }} +""" + + +class TestDatabaseRole: + """ + This test addresses https://github.com/dbt-labs/dbt-snowflake/issues/1151 + + While dbt-snowflake does not manage database roles (it only manages account roles, + it still needs to account for them so that it doesn't try to revoke them. + """ + + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": SEED} + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": MODEL} + + @pytest.fixture(scope="class") + def project_config_update(self): + # grant to the test role even though this role already has these permissions + # this triggers syncing grants since `apply_grants` first looks for a grants config + return {"models": {"+grants": {"select": [os.getenv("SNOWFLAKE_TEST_ROLE")]}}} + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project, prefix): + """ + Create a database role with access to the model we're about to create. + The existence of this database role triggered the bug as dbt-snowflake attempts + to revoke it if the user also provides a grants config. + """ + role = f"BLOCKING_DB_ROLE_{prefix}" + project.run_sql(f"CREATE DATABASE ROLE {role}") + sql = f""" + GRANT + ALL PRIVILEGES ON FUTURE TABLES + IN SCHEMA {project.test_schema} + TO DATABASE ROLE {role} + """ + project.run_sql(sql) + yield + project.run_sql(f"DROP DATABASE ROLE {role}") + + def test_database_role(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + # run a second time to trigger revoke on an incremental update + # this originally failed, demonstrating the bug + run_dbt(["run"]) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_jwt.py b/dbt-snowflake/tests/functional/auth_tests/test_jwt.py new file mode 100644 index 000000000..fbe8e20e6 --- /dev/null +++ b/dbt-snowflake/tests/functional/auth_tests/test_jwt.py @@ -0,0 +1,91 @@ +""" +Please follow the instructions in test_oauth.py for instructions on how to set up +the security integration required to retrieve a JWT from Snowflake. +""" + +import pytest +import os +from dbt.tests.util import run_dbt, check_relations_equal + +from dbt.adapters.snowflake import SnowflakeCredentials + +_MODELS__MODEL_1_SQL = """ +select 1 as id +""" + + +_MODELS__MODEL_2_SQL = """ +select 2 as id +""" + + +_MODELS__MODEL_3_SQL = """ +select * from {{ ref('model_1') }} +union all +select * from {{ ref('model_2') }} +""" + + +_MODELS__MODEL_4_SQL = """ +select 1 as id +union all +select 2 as id +""" + + +class TestSnowflakeJWT: + """Tests that setting authenticator: jwt allows setting token to a plain JWT + that will be passed into the Snowflake connection without modification.""" + + @pytest.fixture(scope="class", autouse=True) + def access_token(self): + """Because JWTs are short-lived, we need to get a fresh JWT via the refresh + token flow before running the test. + + This fixture leverages the existing SnowflakeCredentials._get_access_token + method to retrieve a valid JWT from Snowflake. + """ + client_id = os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_ID") + client_secret = os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET") + refresh_token = os.getenv("SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN") + + credentials = SnowflakeCredentials( + account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + database="", + schema="", + authenticator="oauth", + oauth_client_id=client_id, + oauth_client_secret=client_secret, + token=refresh_token, + ) + + yield credentials._get_access_token() + + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self, access_token): + """A dbt_profile that has authenticator set to JWT, and token set to + a JWT accepted by Snowflake. Also omits the user, as the user attribute + is optional when the authenticator is set to JWT. + """ + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + "authenticator": "jwt", + "token": access_token, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_1.sql": _MODELS__MODEL_1_SQL, + "model_2.sql": _MODELS__MODEL_2_SQL, + "model_3.sql": _MODELS__MODEL_3_SQL, + "model_4.sql": _MODELS__MODEL_4_SQL, + } + + def test_snowflake_basic(self, project): + run_dbt() + check_relations_equal(project.adapter, ["MODEL_3", "MODEL_4"]) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_key_pair.py b/dbt-snowflake/tests/functional/auth_tests/test_key_pair.py new file mode 100644 index 000000000..6d3254f33 --- /dev/null +++ b/dbt-snowflake/tests/functional/auth_tests/test_key_pair.py @@ -0,0 +1,26 @@ +import os + +from dbt.tests.util import run_dbt +import pytest + + +class TestKeyPairAuth: + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self): + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "user": os.getenv("SNOWFLAKE_TEST_USER"), + "private_key": os.getenv("SNOWFLAKE_TEST_PRIVATE_KEY"), + "private_key_passphrase": os.getenv("SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + } + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": "select 1 as id"} + + def test_connection(self, project): + run_dbt() diff --git a/dbt-snowflake/tests/functional/auth_tests/test_oauth.py b/dbt-snowflake/tests/functional/auth_tests/test_oauth.py new file mode 100644 index 000000000..c8986763e --- /dev/null +++ b/dbt-snowflake/tests/functional/auth_tests/test_oauth.py @@ -0,0 +1,90 @@ +""" +The first time using an account for testing, you should run this: + +``` +CREATE OR REPLACE SECURITY INTEGRATION DBT_INTEGRATION_TEST_OAUTH + TYPE = OAUTH + ENABLED = TRUE + OAUTH_CLIENT = CUSTOM + OAUTH_CLIENT_TYPE = 'CONFIDENTIAL' + OAUTH_REDIRECT_URI = 'http://localhost:8080' + oauth_issue_refresh_tokens = true + OAUTH_ALLOW_NON_TLS_REDIRECT_URI = true + BLOCKED_ROLES_LIST = + oauth_refresh_token_validity = 7776000; +``` + + +Every month (or any amount <90 days): + +Run `select SYSTEM$SHOW_OAUTH_CLIENT_SECRETS('DBT_INTEGRATION_TEST_OAUTH');` + +The only row/column of output should be a json blob, it goes (within single +quotes!) as the second argument to the server script: + +python scripts/werkzeug-refresh-token.py ${acount_name} '${json_blob}' + +Open http://localhost:8080 + +Log in as the test user, get a response page with some environment variables. +Update CI providers and test.env with the new values (If you kept the security +integration the same, just the refresh token changed) +""" + +import os +from dbt.tests.util import check_relations_equal, run_dbt +import pytest + + +_MODELS__MODEL_1_SQL = """ +select 1 as id +""" + + +_MODELS__MODEL_2_SQL = """ +select 2 as id +""" + + +_MODELS__MODEL_3_SQL = """ +select * from {{ ref('model_1') }} +union all +select * from {{ ref('model_2') }} +""" + + +_MODELS__MODEL_4_SQL = """ +select 1 as id +union all +select 2 as id +""" + + +class TestSnowflakeOauth: + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self): + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "user": os.getenv("SNOWFLAKE_TEST_USER"), + "oauth_client_id": os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_ID"), + "oauth_client_secret": os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET"), + "token": os.getenv("SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + "authenticator": "oauth", + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_1.sql": _MODELS__MODEL_1_SQL, + "model_2.sql": _MODELS__MODEL_2_SQL, + "model_3.sql": _MODELS__MODEL_3_SQL, + "model_4.sql": _MODELS__MODEL_4_SQL, + } + + def test_snowflake_basic(self, project): + run_dbt() + check_relations_equal(project.adapter, ["MODEL_3", "MODEL_4"]) diff --git a/dbt-snowflake/tests/functional/generic_test_tests/__init__.py b/dbt-snowflake/tests/functional/generic_test_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/functional/generic_test_tests/_files.py b/dbt-snowflake/tests/functional/generic_test_tests/_files.py new file mode 100644 index 000000000..a9743e43e --- /dev/null +++ b/dbt-snowflake/tests/functional/generic_test_tests/_files.py @@ -0,0 +1,90 @@ +SCHEMA__CONTROL = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null +""" + + +SCHEMA__EXPLICIT_WAREHOUSE = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__NOT_NULL = """ +version: 2 +models: + - name: facts + columns: + - name: value + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__RELATIONSHIPS = """ +version: 2 +models: + - name: facts + columns: + - name: color + data_tests: + - relationships: + to: ref('my_colors') + field: color + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__ACCEPTED_VALUES = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - accepted_values: + values: ['blue', 'red', 'green'] + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SEED__COLORS = """ +color +blue +green +red +yellow +""".strip() + + +# record 10 is missing a value +# record 7 has a color that's not on COLORS +SEED__FACTS = """ +id,color,value +1,blue,10 +2,red,20 +3,green,30 +4,yellow,40 +5,blue,50 +6,red,60 +7,orange,70 +8,green,80 +9,yellow,90 +10,blue, +""".strip() diff --git a/dbt-snowflake/tests/functional/generic_test_tests/_models.py b/dbt-snowflake/tests/functional/generic_test_tests/_models.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/functional/generic_test_tests/_schemas.py b/dbt-snowflake/tests/functional/generic_test_tests/_schemas.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/functional/generic_test_tests/test_generic_tests.py b/dbt-snowflake/tests/functional/generic_test_tests/test_generic_tests.py new file mode 100644 index 000000000..a653a363b --- /dev/null +++ b/dbt-snowflake/tests/functional/generic_test_tests/test_generic_tests.py @@ -0,0 +1,54 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.generic_test_tests import _files + + +class TestWarehouseConfig: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "colors.csv": _files.SEED__COLORS, + "facts.csv": _files.SEED__FACTS, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + +class TestWarehouseConfigControl(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__CONTROL} + + def test_expected_warehouse(self, project): + results, logs = run_dbt_and_capture(["test"]) + assert len(results) == 1 + + +class TestWarehouseConfigExplicitWarehouse(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__EXPLICIT_WAREHOUSE} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"]) + assert "use warehouse " in logs + + +class TestWarehouseConfigNotNull(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__NOT_NULL} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"], expect_pass=False) + assert "use warehouse " in logs diff --git a/dbt-snowflake/tests/functional/iceberg/models.py b/dbt-snowflake/tests/functional/iceberg/models.py new file mode 100644 index 000000000..e6da6aca4 --- /dev/null +++ b/dbt-snowflake/tests/functional/iceberg/models.py @@ -0,0 +1,148 @@ +_MODEL_BASIC_TABLE_MODEL = """ +{{ + config( + materialized = "table", + cluster_by=['id'], + ) +}} +select 1 as id +""" + +_MODEL_BASIC_ICEBERG_MODEL = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + base_location_subpath="subpath", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format='iceberg', + external_volume='s3_iceberg_snow', +) }} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH = """ +{{ + config( + transient = "transient", + materialized = "dynamic_table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH = """ +{{ + config( + transient = "true", + materialized = "dynamic_table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + base_location_subpath='subpath', + ) +}} + +select * from {{ ref('first_table') }} +""" + + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format='iceberg', + external_volume='s3_iceberg_snow', + base_location_subpath='subpath', +) }} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BUILT_ON_ICEBERG_TABLE = """ +{{ + config( + materialized = "table", + ) +}} +select * from {{ ref('iceberg_table') }} +""" + +_MODEL_TABLE_BEFORE_SWAP = """ +{{ + config( + materialized = "table", + ) +}} +select 1 as id +""" + +_MODEL_VIEW_BEFORE_SWAP = """ +select 1 as id +""" + +_MODEL_TABLE_FOR_SWAP_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} +select 1 as id +""" diff --git a/dbt-snowflake/tests/functional/iceberg/test_incremental_models.py b/dbt-snowflake/tests/functional/iceberg/test_incremental_models.py new file mode 100644 index 000000000..35ccdcd89 --- /dev/null +++ b/dbt-snowflake/tests/functional/iceberg/test_incremental_models.py @@ -0,0 +1,169 @@ +import pytest +import time + +from pathlib import Path + +from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file + + +_SEED_INCREMENTAL_STRATEGIES = """ +world_id,world_name,boss +1,Yoshi's Island,Iggy +2,Donut Plains,Morton +3,Vanilla Dome,Lemmy +4,Cookie Mountain,Temmy +5,Forest of Illusion,Roy +""".strip() + +_MODEL_BASIC_TABLE_MODEL = """ +{{ + config( + materialized = "table", + ) +}} +select * from {{ ref('seed') }} +""" + +_MODEL_INCREMENTAL_ICEBERG_BASE = """ +{{{{ + config( + materialized='incremental', + table_format='iceberg', + incremental_strategy='{strategy}', + unique_key="world_id", + external_volume = "s3_iceberg_snow", + on_schema_change = "sync_all_columns" + ) +}}}} +select * from {{{{ ref('upstream_table') }}}} + +{{% if is_incremental() %}} +where world_id > 2 +{{% endif %}} +""" + +_MODEL_INCREMENTAL_ICEBERG_APPEND = _MODEL_INCREMENTAL_ICEBERG_BASE.format(strategy="append") +_MODEL_INCREMENTAL_ICEBERG_MERGE = _MODEL_INCREMENTAL_ICEBERG_BASE.format(strategy="merge") +_MODEL_INCREMENTAL_ICEBERG_DELETE_INSERT = _MODEL_INCREMENTAL_ICEBERG_BASE.format( + strategy="delete+insert" +) + + +_QUERY_UPDATE_UPSTREAM_TABLE = """ +UPDATE {database}.{schema}.upstream_table set world_name = 'Twin Bridges', boss = 'Ludwig' where world_id = 4; +""" + +_QUERY_UPDATE_UPSTREAM_TABLE_NO_EFFECT = """ +UPDATE {database}.{schema}.upstream_table set world_name = 'Doughnut Plains' where world_id = 2; +""" + + +class TestIcebergIncrementalStrategies: + append: str = f"append_{hash(time.time())}" + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEED_INCREMENTAL_STRATEGIES, + } + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + yield + + @pytest.fixture(scope="class") + def models(self): + return { + "upstream_table.sql": _MODEL_BASIC_TABLE_MODEL, + f"{self.append}.sql": _MODEL_INCREMENTAL_ICEBERG_APPEND, + "merge.sql": _MODEL_INCREMENTAL_ICEBERG_MERGE, + "delete_insert.sql": _MODEL_INCREMENTAL_ICEBERG_DELETE_INSERT, + } + + def __check_correct_operations(self, model_name, /, rows_affected, status="SUCCESS"): + run_results = run_dbt( + ["show", "--inline", f"select * from {{{{ ref('{model_name}') }}}} where world_id = 4"] + ) + assert run_results[0].adapter_response["rows_affected"] == rows_affected + assert run_results[0].adapter_response["code"] == status + + if "append" not in model_name: + run_results, stdout = run_dbt_and_capture( + [ + "show", + "--inline", + f"select * from {{{{ ref('{model_name}') }}}} where world_id = 2", + ] + ) + run_results[0].adapter_response["rows_affected"] == 1 + assert "Doughnut" not in stdout + + def test_incremental_strategies_with_update(self, project, setup_class): + run_results = run_dbt() + assert len(run_results) == 4 + + project.run_sql( + _QUERY_UPDATE_UPSTREAM_TABLE.format( + database=project.database, schema=project.test_schema + ) + ) + project.run_sql( + _QUERY_UPDATE_UPSTREAM_TABLE_NO_EFFECT.format( + database=project.database, schema=project.test_schema + ) + ) + + run_results = run_dbt(["run", "-s", self.append, "merge", "delete_insert"]) + assert len(run_results) == 3 + + self.__check_correct_operations(self.append, rows_affected=2) + self.__check_correct_operations("merge", rows_affected=1) + self.__check_correct_operations("delete_insert", rows_affected=1) + + +class TestIcebergIncrementalOnSchemaChangeMutatesRelations: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEED_INCREMENTAL_STRATEGIES, + } + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + @pytest.fixture(scope="class") + def models(self): + return { + "upstream_table.sql": _MODEL_BASIC_TABLE_MODEL, + "merge.sql": _MODEL_INCREMENTAL_ICEBERG_MERGE, + } + + def test_sync_and_append_semantics(self, project, setup_class): + model_file = project.project_root / Path("models") / Path("merge.sql") + sql = f"show columns in {project.database}.{project.test_schema}.merge;" + column_names = [column[2] for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 3 + + write_file(_MODEL_INCREMENTAL_ICEBERG_MERGE.replace("*", "*, 1 as new_column"), model_file) + run_dbt() + column_names = [column[2].lower() for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 4 + assert "new_column" in column_names + + write_file(_MODEL_INCREMENTAL_ICEBERG_MERGE, model_file) + run_dbt() + column_names = [column[2].lower() for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 3 + assert "new_column" not in column_names diff --git a/dbt-snowflake/tests/functional/iceberg/test_table_basic.py b/dbt-snowflake/tests/functional/iceberg/test_table_basic.py new file mode 100644 index 000000000..faf4b34f7 --- /dev/null +++ b/dbt-snowflake/tests/functional/iceberg/test_table_basic.py @@ -0,0 +1,68 @@ +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, rm_file, write_file + +from tests.functional.iceberg.models import ( + _MODEL_BASIC_TABLE_MODEL, + _MODEL_BASIC_ICEBERG_MODEL, + _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH, + _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, + _MODEL_BUILT_ON_ICEBERG_TABLE, + _MODEL_TABLE_BEFORE_SWAP, + _MODEL_VIEW_BEFORE_SWAP, + _MODEL_TABLE_FOR_SWAP_ICEBERG, +) + + +class TestIcebergTableBuilds: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def models(self): + return { + "first_table.sql": _MODEL_BASIC_TABLE_MODEL, + "iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL, + "iceberg_tableb.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH, + "iceberg_tablec.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH, + "table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE, + "dynamic_table.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL, + "dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH, + "dynamic_tablec.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH, + "dynamic_tabled.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, + } + + def test_iceberg_tables_build_and_can_be_referred(self, project): + run_results = run_dbt() + assert len(run_results) == 9 + + +class TestIcebergTableTypeBuildsOnExistingTable: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.mark.parametrize("start_model", [_MODEL_TABLE_BEFORE_SWAP, _MODEL_VIEW_BEFORE_SWAP]) + def test_changing_model_types(self, project, start_model): + model_file = project.project_root / Path("models") / Path("my_model.sql") + + write_file(start_model, model_file) + run_results = run_dbt() + assert len(run_results) == 1 + + rm_file(model_file) + write_file(_MODEL_TABLE_FOR_SWAP_ICEBERG, model_file) + run_results = run_dbt() + assert len(run_results) == 1 + + rm_file(model_file) + write_file(start_model, model_file) + run_results = run_dbt() + assert len(run_results) == 1 diff --git a/dbt-snowflake/tests/functional/override_database/test_override_database.py b/dbt-snowflake/tests/functional/override_database/test_override_database.py new file mode 100644 index 000000000..a677ca633 --- /dev/null +++ b/dbt-snowflake/tests/functional/override_database/test_override_database.py @@ -0,0 +1,209 @@ +import pytest +import os +from dbt.tests.util import run_dbt, check_relations_equal_with_relations + +_MODELS__VIEW_1_SQL = """ +{# + We are running against a database that must be quoted. + These calls ensure that we trigger an error if we're failing to quote at parse-time +#} +{% do adapter.already_exists(this.schema, this.table) %} +{% do adapter.get_relation(this.database, this.schema, this.table) %} +select * from {{ ref('seed') }} +""" + +_MODELS__VIEW_2_SQL = """ + {{ config(database=var('alternate_db')) }} +select * from {{ ref('seed') }} +""" + +_MODELS__SUBFOLDER__VIEW_3_SQL = """ +select * from {{ ref('seed') }} +""" + +_MODELS__SUBFOLDER__VIEW_4_SQL = """ +{{ + config(database=var('alternate_db')) +}} + +select * from {{ ref('seed') }} +""" + +_SEEDS__SEED_CSV = """id,name +1,a +2,b +3,c +4,d +5,e +""" + +ALT_DATABASE = os.getenv("SNOWFLAKE_TEST_ALT_DATABASE") + + +class BaseOverrideDatabaseSnowflake: + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__SEED_CSV} + + @pytest.fixture(scope="class") + def models(self): + return { + "view_1.sql": _MODELS__VIEW_1_SQL, + "view_2.sql": _MODELS__VIEW_2_SQL, + "subfolder": { + "view_3.sql": _MODELS__SUBFOLDER__VIEW_3_SQL, + "view_4.sql": _MODELS__SUBFOLDER__VIEW_4_SQL, + }, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "vars": { + "alternate_db": ALT_DATABASE, + }, + "quoting": { + "database": True, + }, + } + + @pytest.fixture(scope="function") + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=ALT_DATABASE, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + @staticmethod + def cap_name(name): + return name.upper() + + +class TestModelOverrideSnowflake(BaseOverrideDatabaseSnowflake): + def run_database_override(self, project): + run_dbt(["seed"]) + result = run_dbt(["run"]) + assert len(result) == 4 + check_relations_equal_with_relations( + project.adapter, + [ + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("seed") + ), + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("view_1") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_2"), + ), + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("view_3") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_4"), + ), + ], + ) + + def test_snowflake_database_override(self, project, clean_up): + self.run_database_override(project) + + +class TestProjectSeedOverrideSnowflake(BaseOverrideDatabaseSnowflake): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "vars": { + "alternate_db": ALT_DATABASE, + }, + "seeds": {"database": ALT_DATABASE}, + } + + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + check_relations_equal_with_relations( + project.adapter, + [ + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("seed"), + ), + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("view_1") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_2"), + ), + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("view_3") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_4"), + ), + ], + ) + + def test_snowflake_database_override(self, project, clean_up): + self.run_database_override(project) + + +class BaseProjectModelOverrideSnowflake(BaseOverrideDatabaseSnowflake): + def run_database_override(self, project): + run_dbt(["seed"]) + result = run_dbt(["run"]) + assert len(result) == 4 + check_relations_equal_with_relations( + project.adapter, + [ + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("seed") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_1"), + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_2"), + ), + project.adapter.Relation.create( + schema=project.test_schema, identifier=self.cap_name("view_3") + ), + project.adapter.Relation.create( + database=ALT_DATABASE, + schema=project.test_schema, + identifier=self.cap_name("view_4"), + ), + ], + ) + + +class TestProjectModelOverrideSnowflake(BaseProjectModelOverrideSnowflake): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "database": ALT_DATABASE, + "test": {"subfolder": {"database": "{{ target.database }}"}}, + }, + "vars": { + "alternate_db": ALT_DATABASE, + }, + } + + def test_snowflake_database_override(self, project, clean_up): + self.run_database_override(project) diff --git a/dbt-snowflake/tests/functional/query_tag/test_query_tags.py b/dbt-snowflake/tests/functional/query_tag/test_query_tags.py new file mode 100644 index 000000000..4ddafabb2 --- /dev/null +++ b/dbt-snowflake/tests/functional/query_tag/test_query_tags.py @@ -0,0 +1,129 @@ +import pytest +from dbt.tests.util import run_dbt + + +snapshots__snapshot_query_tag_sql = """ +{% snapshot snapshot_query_tag %} + {{ + config( + target_database=database, + target_schema=schema, + unique_key='id', + strategy='check', + check_cols=['color'], + ) + }} + select 1 as id, 'blue' as color +{% endsnapshot %} +""" + + +models__table_model_query_tag_sql = """ +{{ config(materialized = 'table') }} +select 1 as id +""" + + +models__models_config_yml = """ +version: 2 + +models: + - name: view_model_query_tag + columns: + - name: id + data_tests: + - unique +""" + + +models__view_model_query_tag_sql = """ +{{ config(materialized = 'view') }} +select 1 as id +""" + + +models__incremental_model_query_tag_sql = """ +{{ config(materialized = 'incremental', unique_key = 'id') }} +select 1 as id +""" + + +macros__check_tag_sql = """ +{% macro check_query_tag() %} + + {% if execute %} + {% set query_tag = get_current_query_tag() %} + {% if query_tag != var("query_tag") %} + {{ exceptions.raise_compiler_error("Query tag not used!") }} + {% endif %} + {% endif %} + +{% endmacro %} +""" + + +seeds__seed_query_tag_csv = """id +1 +""".strip() + + +class TestQueryTag: + @pytest.fixture(scope="class") + def models(self): + return { + "table_model_query_tag.sql": models__table_model_query_tag_sql, + "view_model_query_tag.sql": models__view_model_query_tag_sql, + "incremental_model_query_tag.sql": models__incremental_model_query_tag_sql, + "models_config.yml": models__models_config_yml, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return {"snapshot_query_tag.sql": snapshots__snapshot_query_tag_sql} + + @pytest.fixture(scope="class") + def macros(self): + return {"check_tag.sql": macros__check_tag_sql} + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed_query_tag.csv": seeds__seed_query_tag_csv} + + @pytest.fixture(scope="class") + def project_config_update(self, prefix): + return { + "config-version": 2, + "models": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "seeds": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "snapshots": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "tests": {"test": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}}, + } + + def build_all_with_query_tags(self, project, prefix): + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) + + def test_snowflake_query_tag(self, project, prefix): + self.build_all_with_query_tags(project, prefix) + self.build_all_with_query_tags(project, prefix) + + +class TestSnowflakeProfileQueryTag: + @pytest.fixture(scope="class") + def models(self): + return { + "table_model_query_tag.sql": models__table_model_query_tag_sql, + "view_model_query_tag.sql": models__view_model_query_tag_sql, + "incremental_model_query_tag.sql": models__incremental_model_query_tag_sql, + "models_config.yml": models__models_config_yml, + } + + @pytest.fixture(scope="class") + def profiles_config_update(self, prefix): + return {"query_tag": prefix} + + def build_all_with_query_tags(self, project, prefix): + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) + + def test_snowflake_query_tag(self, project, prefix): + self.build_all_with_query_tags(project, prefix) + self.build_all_with_query_tags(project, prefix) diff --git a/dbt-snowflake/tests/functional/redact_log_values/test_duplicate_key_not_in_exceptions.py b/dbt-snowflake/tests/functional/redact_log_values/test_duplicate_key_not_in_exceptions.py new file mode 100644 index 000000000..e32f08c15 --- /dev/null +++ b/dbt-snowflake/tests/functional/redact_log_values/test_duplicate_key_not_in_exceptions.py @@ -0,0 +1,33 @@ +import pytest + +from dbt.tests.util import ( + run_dbt, +) + +_MODELS__view = """ +{{ config( + materialized='table', +) }} + +with dupes as ( + select 'foo' as key, 1 as value + union all + select 'foo' as key, 2 as value +) + +select + object_agg(key, value) as agg +from dupes +""" + + +class TestDuplicateKeyNotInExceptions: + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": _MODELS__view} + + def test_row_values_were_scrubbed_from_duplicate_merge_exception(self, project): + result = run_dbt(["run", "-s", "model"], expect_pass=False) + assert len(result) == 1 + assert "Duplicate field key '[redacted]'" in result[0].message + assert "'foo'" not in result[0].message diff --git a/dbt-snowflake/tests/functional/redact_log_values/test_row_values_not_in_exceptions.py b/dbt-snowflake/tests/functional/redact_log_values/test_row_values_not_in_exceptions.py new file mode 100644 index 000000000..ec4fe2348 --- /dev/null +++ b/dbt-snowflake/tests/functional/redact_log_values/test_row_values_not_in_exceptions.py @@ -0,0 +1,34 @@ +import pytest + +from dbt.tests.util import ( + run_dbt, +) + +_MODELS__incremental_model = """ +{{ config( + materialized='incremental', + unique_key='id' +) }} + +with data as ( + SELECT $1 id, $2 name FROM ( + VALUES (1, 'one'), (2, 'two'), (3, 'three'), (1, 'one') + ) +) +select * from data +""" + + +class TestRowValuesNotInExceptions: + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": _MODELS__incremental_model} + + def test_row_values_were_scrubbed_from_duplicate_merge_exception(self, project): + result = run_dbt(["run", "-s", "model"]) + assert len(result) == 1 + + result = run_dbt(["run", "-s", "model"], expect_pass=False) + assert len(result) == 1 + assert "Row Values: [redacted]" in result[0].message + assert "'one'" not in result[0].message diff --git a/dbt-snowflake/tests/functional/relation_tests/__init__.py b/dbt-snowflake/tests/functional/relation_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/functional/relation_tests/base.py b/dbt-snowflake/tests/functional/relation_tests/base.py new file mode 100644 index 000000000..d08a6945b --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/base.py @@ -0,0 +1,75 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + + +SEED = """ +id +0 +1 +2 +""".strip() + + +TABLE = """ +{{ config(materialized="table") }} +select * from {{ ref('my_seed') }} +""" + + +VIEW = """ +{{ config(materialized="view") }} +select * from {{ ref('my_seed') }} +""" + + +MACRO__GET_CREATE_BACKUP_SQL = """ +{% macro test__get_create_backup_sql(database, schema, identifier, relation_type) -%} + {%- set relation = adapter.Relation.create(database=database, schema=schema, identifier=identifier, type=relation_type) -%} + {% call statement('test__get_create_backup_sql') -%} + {{ get_create_backup_sql(relation) }} + {%- endcall %} +{% endmacro %}""" + + +MACRO__GET_RENAME_INTERMEDIATE_SQL = """ +{% macro test__get_rename_intermediate_sql(database, schema, identifier, relation_type) -%} + {%- set relation = adapter.Relation.create(database=database, schema=schema, identifier=identifier, type=relation_type) -%} + {% call statement('test__get_rename_intermediate_sql') -%} + {{ get_rename_intermediate_sql(relation) }} + {%- endcall %} +{% endmacro %}""" + + +class RelationOperation: + @pytest.fixture(scope="class") + def seeds(self): + yield {"my_seed.csv": SEED} + + @pytest.fixture(scope="class") + def models(self): + yield { + "my_table.sql": TABLE, + "my_table__dbt_tmp.sql": TABLE, + "my_view.sql": VIEW, + "my_view__dbt_tmp.sql": VIEW, + } + + @pytest.fixture(scope="class") + def macros(self): + yield { + "test__get_create_backup_sql.sql": MACRO__GET_CREATE_BACKUP_SQL, + "test__get_rename_intermediate_sql.sql": MACRO__GET_RENAME_INTERMEDIATE_SQL, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + def assert_operation(self, project, operation, args, expected_statement): + results, logs = run_dbt_and_capture( + ["--debug", "run-operation", operation, "--args", str(args)] + ) + assert len(results) == 1 + assert expected_statement in logs diff --git a/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/__init__.py b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/models.py b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/models.py new file mode 100644 index 000000000..57d83f968 --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/models.py @@ -0,0 +1,112 @@ +SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +EXPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='AUTO', +) }} +select * from {{ ref('my_seed') }} +""" + +IMPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_DOWNSTREAM = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='DOWNSTREAM', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_ALTER = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='5 minutes', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_REPLACE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='FULL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE_ALTER = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='5 minutes', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE_REPLACE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='FULL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_basic.py b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_basic.py new file mode 100644 index 000000000..8cdf59ebc --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_basic.py @@ -0,0 +1,89 @@ +import pytest + +from dbt.tests.util import assert_message_in_logs, run_dbt, run_dbt_and_capture + +from tests.functional.relation_tests.dynamic_table_tests import models +from tests.functional.utils import query_relation_type + + +class TestBasic: + iceberg: bool = False + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + my_models = { + "my_dynamic_table.sql": models.DYNAMIC_TABLE, + "my_dynamic_table_downstream.sql": models.DYNAMIC_TABLE_DOWNSTREAM, + } + if self.iceberg: + my_models.update( + { + "my_dynamic_iceberg_table.sql": models.DYNAMIC_ICEBERG_TABLE, + } + ) + yield my_models + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + def test_dynamic_table_full_refresh(self, project): + run_dbt(["run", "--full-refresh"]) + assert query_relation_type(project, "my_dynamic_table") == "dynamic_table" + assert query_relation_type(project, "my_dynamic_table_downstream") == "dynamic_table" + if self.iceberg: + assert query_relation_type(project, "my_dynamic_iceberg_table") == "dynamic_table" + + +class TestBasicIcebergOn(TestBasic): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + +class TestAutoConfigDoesntFullRefresh: + """ + AUTO refresh_strategy will be compared accurately with both INCREMENTAL and FULL. + https://github.com/dbt-labs/dbt-snowflake/issues/1267 + """ + + DT_NAME = "my_dynamic_table" + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + f"explicit_{self.DT_NAME}.sql": models.EXPLICIT_AUTO_DYNAMIC_TABLE, + f"implicit_{self.DT_NAME}.sql": models.IMPLICIT_AUTO_DYNAMIC_TABLE, + } + + @pytest.mark.parametrize("test_dt", [f"explicit_{DT_NAME}", f"implicit_{DT_NAME}"]) + def test_auto_config_doesnt_full_refresh(self, project, test_dt): + model_qualified_name = f"{project.database}.{project.test_schema}.{test_dt}" + + run_dbt(["seed"]) + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs) + assert_message_in_logs("refresh_mode = AUTO", logs) + + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs, False) + assert_message_in_logs( + f"create or replace dynamic table {model_qualified_name}", logs, False + ) + assert_message_in_logs("refresh_mode = AUTO", logs, False) + assert_message_in_logs( + f"No configuration changes were identified on: `{model_qualified_name}`. Continuing.", + logs, + ) diff --git a/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py new file mode 100644 index 000000000..f389344e0 --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py @@ -0,0 +1,177 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.relation_tests.dynamic_table_tests import models +from tests.functional.utils import describe_dynamic_table, update_model + + +class Changes: + iceberg: bool = False + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + yield {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + my_models = { + "dynamic_table_alter.sql": models.DYNAMIC_TABLE, + "dynamic_table_replace.sql": models.DYNAMIC_TABLE, + } + if self.iceberg: + my_models.update( + { + "dynamic_table_iceberg_alter.sql": models.DYNAMIC_ICEBERG_TABLE, + "dynamic_table_iceberg_replace.sql": models.DYNAMIC_ICEBERG_TABLE, + } + ) + yield my_models + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + yield + project.run_sql(f"drop schema if exists {project.test_schema} cascade") + + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, project, setup_class): + # make sure the model in the data reflects the files each time + run_dbt(["run", "--full-refresh"]) + self.assert_changes_are_not_applied(project) + + update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE_ALTER) + update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE_REPLACE) + if self.iceberg: + update_model( + project, "dynamic_table_iceberg_alter", models.DYNAMIC_ICEBERG_TABLE_ALTER + ) + update_model( + project, "dynamic_table_iceberg_replace", models.DYNAMIC_ICEBERG_TABLE_REPLACE + ) + + yield + + update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE) + update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE) + if self.iceberg: + update_model(project, "dynamic_table_iceberg_alter", models.DYNAMIC_ICEBERG_TABLE) + update_model(project, "dynamic_table_iceberg_replace", models.DYNAMIC_ICEBERG_TABLE) + + def assert_changes_are_applied(self, project): + altered = describe_dynamic_table(project, "dynamic_table_alter") + assert altered.snowflake_warehouse == "DBT_TESTING" + assert altered.target_lag == "5 minutes" # this updated + assert altered.refresh_mode == "INCREMENTAL" + + replaced = describe_dynamic_table(project, "dynamic_table_replace") + assert replaced.snowflake_warehouse == "DBT_TESTING" + assert replaced.target_lag == "2 minutes" + assert replaced.refresh_mode == "FULL" # this updated + + if self.iceberg: + altered_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_alter") + assert altered_iceberg.snowflake_warehouse == "DBT_TESTING" + assert altered_iceberg.target_lag == "5 minutes" # this updated + assert altered_iceberg.refresh_mode == "INCREMENTAL" + + replaced_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_replace") + assert replaced_iceberg.snowflake_warehouse == "DBT_TESTING" + assert replaced_iceberg.target_lag == "2 minutes" + assert replaced_iceberg.refresh_mode == "FULL" # this updated + + def assert_changes_are_not_applied(self, project): + altered = describe_dynamic_table(project, "dynamic_table_alter") + assert altered.snowflake_warehouse == "DBT_TESTING" + assert altered.target_lag == "2 minutes" # this would have updated, but didn't + assert altered.refresh_mode == "INCREMENTAL" + + replaced = describe_dynamic_table(project, "dynamic_table_replace") + assert replaced.snowflake_warehouse == "DBT_TESTING" + assert replaced.target_lag == "2 minutes" + assert replaced.refresh_mode == "INCREMENTAL" # this would have updated, but didn't + + if self.iceberg: + altered_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_alter") + assert altered_iceberg.snowflake_warehouse == "DBT_TESTING" + assert altered_iceberg.target_lag == "2 minutes" # this would have updated, but didn't + assert altered_iceberg.refresh_mode == "INCREMENTAL" + + replaced_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_replace") + assert replaced_iceberg.snowflake_warehouse == "DBT_TESTING" + assert replaced_iceberg.target_lag == "2 minutes" + assert ( + replaced_iceberg.refresh_mode == "INCREMENTAL" + ) # this would have updated, but didn't + + def test_full_refresh_is_always_successful(self, project): + # this always passes and always changes the configuration, regardless of on_configuration_change + # and regardless of whether the changes require a replace versus an alter + run_dbt(["run", "--full-refresh"]) + self.assert_changes_are_applied(project) + + +class TestChangesApply(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "apply"}} + + def test_changes_are_applied(self, project): + # this passes and changes the configuration + run_dbt(["run"]) + self.assert_changes_are_applied(project) + + +class TestChangesApplyIcebergOn(TestChangesApply): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "apply"}, + "flags": {"enable_iceberg_materializations": True}, + } + + +class TestChangesContinue(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "continue"}} + + def test_changes_are_not_applied(self, project): + # this passes but does not change the configuration + run_dbt(["run"]) + self.assert_changes_are_not_applied(project) + + +class TestChangesContinueIcebergOn(TestChangesContinue): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "continue"}, + "flags": {"enable_iceberg_materializations": True}, + } + + +class TestChangesFail(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "fail"}} + + def test_changes_are_not_applied(self, project): + # this fails and does not change the configuration + run_dbt(["run"], expect_pass=False) + self.assert_changes_are_not_applied(project) + + +class TestChangesFailIcebergOn(TestChangesFail): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "fail"}, + "flags": {"enable_iceberg_materializations": True}, + } diff --git a/dbt-snowflake/tests/functional/relation_tests/models.py b/dbt-snowflake/tests/functional/relation_tests/models.py new file mode 100644 index 000000000..7b0050d11 --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/models.py @@ -0,0 +1,77 @@ +SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +TABLE = """ +{{ config( + materialized='table', +) }} +select * from {{ ref('my_seed') }} +""" + + +VIEW = """ +{{ config( + materialized='view', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + +ICEBERG_TABLE = """ +{{ config( + materialized='table', + table_format="iceberg", + external_volume="s3_iceberg_snow", +) }} +select * from {{ ref('my_seed') }} +""" + +INCREMENTAL_ICEBERG_TABLE = """ +{{ config( + materialized='incremental', + table_format='iceberg', + incremental_strategy='append', + unique_key="id", + external_volume = "s3_iceberg_snow", +) }} +select * from {{ ref('my_seed') }} +""" + + +INCREMENTAL_TABLE = """ +{{ config( + materialized='incremental', + incremental_strategy='append', + unique_key="id", +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/dbt-snowflake/tests/functional/relation_tests/test_relation_type_change.py b/dbt-snowflake/tests/functional/relation_tests/test_relation_type_change.py new file mode 100644 index 000000000..1024a92ca --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/test_relation_type_change.py @@ -0,0 +1,161 @@ +from dataclasses import dataclass +from itertools import product +from typing import Optional + +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.relation_tests import models +from tests.functional.utils import describe_dynamic_table, query_relation_type, update_model + + +@dataclass +class Model: + model: str + relation_type: str + table_format: Optional[str] = "default" + is_incremental: Optional[bool] = False + + @property + def name(self) -> str: + if self.is_incremental: + name = f"{self.relation_type}_{self.table_format}_incremental" + else: + name = f"{self.relation_type}_{self.table_format}" + return name + + @property + def is_iceberg(self) -> bool: + return self.table_format == "iceberg" + + @property + def is_standard_table(self) -> bool: + return self.relation_type == "table" and not self.is_incremental + + +@dataclass +class Scenario: + initial: Model + final: Model + + @property + def name(self) -> str: + return f"REPLACE_{self.initial.name}__WITH_{self.final.name}" + + @property + def error_message(self) -> str: + return f"Failed when migrating from: {self.initial.name} to: {self.final.name}" + + @property + def uses_iceberg(self) -> bool: + return any([self.initial.is_iceberg, self.final.is_iceberg]) + + +relations = [ + Model(models.VIEW, "view"), + Model(models.TABLE, "table", "default"), + Model(models.INCREMENTAL_TABLE, "table", "default", is_incremental=True), + Model(models.DYNAMIC_TABLE, "dynamic_table", "default"), + Model(models.ICEBERG_TABLE, "table", "iceberg"), + Model(models.INCREMENTAL_ICEBERG_TABLE, "table", "iceberg", is_incremental=True), + Model(models.DYNAMIC_ICEBERG_TABLE, "dynamic_table", "iceberg"), +] +scenarios = [Scenario(*scenario) for scenario in product(relations, relations)] + + +def requires_full_refresh(scenario) -> bool: + return any( + [ + # we can only swap incremental to table and back if both are iceberg + scenario.initial.is_incremental + and scenario.final.is_standard_table + and scenario.initial.table_format != scenario.final.table_format, + scenario.initial.is_standard_table + and scenario.final.is_incremental + and scenario.initial.table_format != scenario.final.table_format, + # we can't swap from an incremental to a dynamic table because the materialization does not handle this case + scenario.initial.relation_type == "dynamic_table" and scenario.final.is_incremental, + ] + ) + + +class TestRelationTypeChange: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": False}} + + @staticmethod + def include(scenario) -> bool: + return not scenario.uses_iceberg and not requires_full_refresh(scenario) + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + f"{scenario.name}.sql": scenario.initial.model + for scenario in scenarios + if self.include(scenario) + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + for scenario in scenarios: + if self.include(scenario): + update_model(project, scenario.name, scenario.final.model) + # allow for dbt to fail so that we can see which scenarios pass and which scenarios fail + try: + run_dbt(["run"], expect_pass=False) + except Exception: + pass + + @pytest.mark.parametrize("scenario", scenarios, ids=[scenario.name for scenario in scenarios]) + def test_replace(self, project, scenario): + if self.include(scenario): + relation_type = query_relation_type(project, scenario.name) + assert relation_type == scenario.final.relation_type, scenario.error_message + if relation_type == "dynamic_table": + dynamic_table = describe_dynamic_table(project, scenario.name) + assert dynamic_table.catalog.table_format == scenario.final.table_format + else: + pytest.skip() + + +class TestRelationTypeChangeFullRefreshRequired(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"enable_iceberg_materializations": False}, + "models": {"full_refresh": True}, + } + + @staticmethod + def include(scenario) -> bool: + return not scenario.uses_iceberg and requires_full_refresh(scenario) + + +class TestRelationTypeChangeIcebergOn(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @staticmethod + def include(scenario) -> bool: + return scenario.uses_iceberg and not requires_full_refresh(scenario) + + +class TestRelationTypeChangeIcebergOnFullRefreshRequired(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"enable_iceberg_materializations": True}, + "models": {"full_refresh": True}, + } + + @staticmethod + def include(scenario) -> bool: + return scenario.uses_iceberg and requires_full_refresh(scenario) diff --git a/dbt-snowflake/tests/functional/relation_tests/test_table.py b/dbt-snowflake/tests/functional/relation_tests/test_table.py new file mode 100644 index 000000000..b4a8709ea --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/test_table.py @@ -0,0 +1,25 @@ +from tests.functional.relation_tests.base import RelationOperation + + +class TestTable(RelationOperation): + + def test_get_create_backup_and_rename_intermediate_sql(self, project): + args = { + "database": project.database, + "schema": project.test_schema, + "identifier": "my_table", + "relation_type": "table", + } + expected_statement = ( + f"alter table {project.database}.{project.test_schema}.my_table " + f"rename to {project.database}.{project.test_schema}.my_table__dbt_backup" + ) + self.assert_operation(project, "test__get_create_backup_sql", args, expected_statement) + + expected_statement = ( + f"alter table {project.database}.{project.test_schema}.my_table__dbt_tmp " + f"rename to {project.database}.{project.test_schema}.my_table" + ) + self.assert_operation( + project, "test__get_rename_intermediate_sql", args, expected_statement + ) diff --git a/dbt-snowflake/tests/functional/relation_tests/test_view.py b/dbt-snowflake/tests/functional/relation_tests/test_view.py new file mode 100644 index 000000000..721455da1 --- /dev/null +++ b/dbt-snowflake/tests/functional/relation_tests/test_view.py @@ -0,0 +1,25 @@ +from tests.functional.relation_tests.base import RelationOperation + + +class TestView(RelationOperation): + + def test_get_create_backup_and_rename_intermediate_sql(self, project): + args = { + "database": project.database, + "schema": project.test_schema, + "identifier": "my_view", + "relation_type": "view", + } + expected_statement = ( + f"alter view {project.database}.{project.test_schema}.my_view " + f"rename to {project.database}.{project.test_schema}.my_view__dbt_backup" + ) + self.assert_operation(project, "test__get_create_backup_sql", args, expected_statement) + + expected_statement = ( + f"alter view {project.database}.{project.test_schema}.my_view__dbt_tmp " + f"rename to {project.database}.{project.test_schema}.my_view" + ) + self.assert_operation( + project, "test__get_rename_intermediate_sql", args, expected_statement + ) diff --git a/dbt-snowflake/tests/functional/snowflake_view_dependency/test_snowflake_view_dependency.py b/dbt-snowflake/tests/functional/snowflake_view_dependency/test_snowflake_view_dependency.py new file mode 100644 index 000000000..4b7948187 --- /dev/null +++ b/dbt-snowflake/tests/functional/snowflake_view_dependency/test_snowflake_view_dependency.py @@ -0,0 +1,104 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal + +_MODELS__DEPENDENT_MODEL_SQL = """ +{% if var('dependent_type', 'view') == 'view' %} + {{ config(materialized='view') }} +{% else %} + {{ config(materialized='table') }} +{% endif %} + +select * from {{ ref('base_table') }} +""" + + +_MODELS__BASE_TABLE_SQL = """ +{{ config(materialized='table') }} +select * + {% if var('add_table_field', False) %} + , 1 as new_field + {% endif %} + +from {{ ref('people') }} +""" + +_SEEDS__PEOPLE_CSV = """id,name +1,Drew +2,Jake +3,Connor +""" + + +class TestSnowflakeLateBindingViewDependency: + @pytest.fixture(scope="class") + def models(self): + return { + "dependent_model.sql": _MODELS__DEPENDENT_MODEL_SQL, + "base_table.sql": _MODELS__BASE_TABLE_SQL, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"people.csv": _SEEDS__PEOPLE_CSV} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + "quoting": {"schema": False, "identifier": False}, + } + + @pytest.fixture(scope="class", autouse=True) + def setup_method(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 2 + check_relations_equal(project.adapter, ["PEOPLE", "BASE_TABLE"]) + check_relations_equal(project.adapter, ["PEOPLE", "DEPENDENT_MODEL"]) + + def check_result(self, project, results, expected_types): + for result in results: + node_name = result.node.name + node_type = result.node.config.materialized + assert node_type == expected_types[node_name] + + """ + Snowflake views are not bound to the relations they select from. A Snowflake view + can have entirely invalid SQL if, for example, the table it selects from is dropped + and recreated with a different schema. In these scenarios, Snowflake will raise an error if: + 1) The view is queried + 2) The view is altered + + dbt's logic should avoid running these types of queries against views in situations + where they _may_ have invalid definitions. These tests assert that views are handled + correctly in various different scenarios + """ + + def test__snowflake__changed_table_schema_for_downstream_view(self, project): + run_dbt(["seed"]) + # Change the schema of base_table, assert that dependent_model doesn't fail + results = run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: view}"]) + assert len(results) == 2 + check_relations_equal(project.adapter, ["BASE_TABLE", "DEPENDENT_MODEL"]) + + """ + This test is similar to the one above, except the downstream model starts as a view, and + then is changed to be a table. This checks that the table materialization does not + errantly rename a view that might have an invalid definition, which would cause an error + """ + + def test__snowflake__changed_table_schema_for_downstream_view_changed_to_table(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + expected_types = {"base_table": "table", "dependent_model": "view"} + # ensure that the model actually was materialized as a view + self.check_result(project, results, expected_types) + results = run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: table}"]) + assert len(results) == 2 + check_relations_equal(project.adapter, ["BASE_TABLE", "DEPENDENT_MODEL"]) + expected_types = {"base_table": "table", "dependent_model": "table"} + # ensure that the model actually was materialized as a table + self.check_result(project, results, expected_types) diff --git a/dbt-snowflake/tests/functional/test_isolated_begin_commit.py b/dbt-snowflake/tests/functional/test_isolated_begin_commit.py new file mode 100644 index 000000000..9e205b7a9 --- /dev/null +++ b/dbt-snowflake/tests/functional/test_isolated_begin_commit.py @@ -0,0 +1,44 @@ +import pytest +from dbt.tests.util import run_dbt_and_capture + +my_model_sql = """ +{{ + config( + materialized = 'table', + post_hook = '{{ my_silly_insert_macro() }}' + ) +}} +select 1 as id, 'blue' as color, current_timestamp as updated_at +""" + +my_macro_sql = """ +{% macro my_silly_insert_macro() %} + {#-- This is a bad pattern! Made obsolete by changes in v0.21 + v1.2 --#} + {% do run_query('begin;') %} + {% set query %} + insert into {{ this }} values (2, 'red', current_timestamp); + {% endset %} + {% do run_query(query) %} + {% do run_query('commit;') %} +{% endmacro %} +""" + + +class TestModelWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "my_macro.sql": my_macro_sql, + } + + def test_isolated_begin_commit(self, project): + # this should succeed / avoid raising an error + results, log_output = run_dbt_and_capture(["run"]) + # but we should see a warning in the logs + assert "WARNING" in log_output and "Explicit transactional logic" in log_output diff --git a/dbt-snowflake/tests/functional/utils.py b/dbt-snowflake/tests/functional/utils.py new file mode 100644 index 000000000..d185e8d2b --- /dev/null +++ b/dbt-snowflake/tests/functional/utils.py @@ -0,0 +1,78 @@ +from typing import Any, Dict, Optional + +from dbt.tests.util import ( + get_connection, + get_model_file, + relation_from_name, + set_model_file, +) + +from dbt.adapters.snowflake.relation_configs import SnowflakeDynamicTableConfig + + +def query_relation_type(project, name: str) -> Optional[str]: + relation = relation_from_name(project.adapter, name) + sql = f""" + select + case table_type + when 'BASE TABLE' then iff(is_dynamic = 'YES', 'dynamic_table', 'table') + when 'VIEW' then 'view' + when 'EXTERNAL TABLE' then 'external_table' + end as relation_type + from information_schema.tables + where table_name like '{relation.identifier.upper()}' + and table_schema like '{relation.schema.upper()}' + and table_catalog like '{relation.database.upper()}' + """ + results = project.run_sql(sql, fetch="all") + + assert len(results) > 0, f"Relation {relation} not found" + assert len(results) == 1, f"Multiple relations found" + + return results[0][0].lower() + + +def query_row_count(project, name: str) -> int: + relation = relation_from_name(project.adapter, name) + sql = f"select count(*) from {relation}" + return project.run_sql(sql, fetch="one")[0] + + +def insert_record(project, name: str, record: Dict[str, Any]): + relation = relation_from_name(project.adapter, name) + column_names = ", ".join(record.keys()) + values = ", ".join( + [f"'{value}'" if isinstance(value, str) else f"{value}" for value in record.values()] + ) + sql = f"insert into {relation} ({column_names}) values ({values})" + project.run_sql(sql) + + +def update_model(project, name: str, model: str) -> str: + relation = relation_from_name(project.adapter, name) + original_model = get_model_file(project, relation) + set_model_file(project, relation, model) + return original_model + + +def describe_dynamic_table(project, name: str) -> Optional[SnowflakeDynamicTableConfig]: + macro = "snowflake__describe_dynamic_table" + dynamic_table = relation_from_name(project.adapter, name) + kwargs = {"relation": dynamic_table} + with get_connection(project.adapter): + results = project.adapter.execute_macro(macro, kwargs=kwargs) + + assert len(results["dynamic_table"].rows) > 0, f"Dynamic table {dynamic_table} not found" + found = len(results["dynamic_table"].rows) + names = ", ".join([table.get("name") for table in results["dynamic_table"].rows]) + assert found == 1, f"Multiple dynamic tables found: {names}" + + return SnowflakeDynamicTableConfig.from_relation_results(results) + + +def refresh_dynamic_table(project, name: str) -> None: + macro = "snowflake__refresh_dynamic_table" + dynamic_table = relation_from_name(project.adapter, name) + kwargs = {"relation": dynamic_table} + with get_connection(project.adapter): + project.adapter.execute_macro(macro, kwargs=kwargs) diff --git a/dbt-snowflake/tests/functional/warehouse_test/test_warehouses.py b/dbt-snowflake/tests/functional/warehouse_test/test_warehouses.py new file mode 100644 index 000000000..268473729 --- /dev/null +++ b/dbt-snowflake/tests/functional/warehouse_test/test_warehouses.py @@ -0,0 +1,135 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal + +import os + +models__override_warehouse_sql = """ +{{ config(snowflake_warehouse=env_var('SNOWFLAKE_TEST_ALT_WAREHOUSE', 'DBT_TEST_ALT'), materialized='table') }} +select current_warehouse() as warehouse +""" + +models__expected_warehouse_sql = """ +{{ config(materialized='table') }} +select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse +""" + +models__invalid_warehouse_sql = """ +{{ config(snowflake_warehouse='DBT_TEST_DOES_NOT_EXIST') }} +select current_warehouse() as warehouse +""" + +project_config_models__override_warehouse_sql = """ +{{ config(materialized='table') }} +select current_warehouse() as warehouse +""" + +project_config_models__expected_warehouse_sql = """ +{{ config(materialized='table') }} +select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse +""" + +project_config_models__warehouse_sql = """ +{{ config(materialized='table') }} +select current_warehouse() as warehouse +""" + + +class TestModelWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "override_warehouse.sql": models__override_warehouse_sql, + "expected_warehouse.sql": models__expected_warehouse_sql, + "invalid_warehouse.sql": models__invalid_warehouse_sql, + } + + def test_snowflake_override_ok(self, project): + run_dbt( + [ + "run", + "--models", + "override_warehouse", + "expected_warehouse", + ] + ) + check_relations_equal(project.adapter, ["OVERRIDE_WAREHOUSE", "EXPECTED_WAREHOUSE"]) + + def test_snowflake_override_noexist(self, project): + run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) + + +class TestConfigWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "override_warehouse.sql": project_config_models__override_warehouse_sql, + "expected_warehouse.sql": project_config_models__expected_warehouse_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "models": { + "test": { + "snowflake_warehouse": os.getenv( + "SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT" + ), + }, + }, + } + + def test_snowflake_override_ok(self, project): + run_dbt( + [ + "run", + "--models", + "override_warehouse", + "expected_warehouse", + ] + ) + check_relations_equal(project.adapter, ["OVERRIDE_WAREHOUSE", "EXPECTED_WAREHOUSE"]) + + +class TestInvalidConfigWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "invalid_warehouse.sql": project_config_models__warehouse_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "models": { + "test": {"snowflake_warehouse": "DBT_TEST_DOES_NOT_EXIST"}, + }, + } + + def test_snowflake_override_invalid(self, project): + result = run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) + assert "Object does not exist, or operation cannot be performed" in result[0].message + + +class TestValidConfigWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "valid_warehouse.sql": project_config_models__warehouse_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "models": { + "test": { + "snowflake_warehouse": "DBT_TESTING", + }, + }, + } + + def test_snowflake_warehouse_valid(self, project): + result = run_dbt(["run", "--models", "valid_warehouse"]) + assert "DBT_TESTING" in result[0].node.config.get("snowflake_warehouse") diff --git a/dbt-snowflake/tests/performance/README.md b/dbt-snowflake/tests/performance/README.md new file mode 100644 index 000000000..02130c5c6 --- /dev/null +++ b/dbt-snowflake/tests/performance/README.md @@ -0,0 +1,6 @@ +# Performance testing + +These tests are not meant to run on a regular basis; instead, they are tools for measuring performance impacts of changes as needed. +We often get requests for reducing processing times, researching why a particular component is taking longer to run than expected, etc. +In the past we have performed one-off analyses to address these requests and documented the results in the relevant PR (when a change is made). +It is more useful to document those analyses in the form of performance tests so that we can easily rerun the analysis at a later date. diff --git a/dbt-snowflake/tests/performance/test_auth_methods.py b/dbt-snowflake/tests/performance/test_auth_methods.py new file mode 100644 index 000000000..ad0b424ab --- /dev/null +++ b/dbt-snowflake/tests/performance/test_auth_methods.py @@ -0,0 +1,132 @@ +""" +Results: + +| method | project_size | reuse_connections | unsafe_skip_rsa_key_validation | duration | +|---------------|--------------|-------------------|--------------------------------|----------| +| User Password | 1,000 | False | - | 234.09s | +| User Password | 1,000 | True | - | 78.34s | +| Key Pair | 1,000 | False | False | 271.47s | +| Key Pair | 1,000 | False | True | 275.73s | +| Key Pair | 1,000 | True | False | 63.69s | +| Key Pair | 1,000 | True | True | 73.45s | + +Notes: +- run locally on MacOS, single threaded +- `unsafe_skip_rsa_key_validation` only applies to the Key Pair auth method +- `unsafe_skip_rsa_key_validation=True` was tested by updating the relevant `cryptography` calls directly as it is not a user configuration +- since the models are all views, time differences should be viewed as absolute differences, e.g.: + - this: (271.47s - 63.69s) / 1,000 models = 208ms improvement + - NOT this: 1 - (63.69s / 271.47s) = 76.7% improvement +""" + +from datetime import datetime +import os + +from dbt.tests.util import run_dbt +import pytest + + +SEED = """ +id,value +1,a +2,b +3,c +""".strip() + + +MODEL = """ +select * from {{ ref("my_seed") }} +""" + + +class Scenario: + """ + Runs a full load test. The test can be configured to run an arbitrary number of models. + + To use this test, configure the test by setting `project_size` and/or `expected_duration`. + """ + + auth_method: str + project_size: int = 1 + reuse_connections: bool = False + + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": SEED} + + @pytest.fixture(scope="class") + def models(self): + return {f"my_model_{i}.sql": MODEL for i in range(self.project_size)} + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + start = datetime.now() + yield + end = datetime.now() + + duration = (end - start).total_seconds() + print(f"Run took: {duration} seconds") + + @pytest.fixture(scope="class") + def dbt_profile_target(self, auth_params): + yield { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + "user": os.getenv("SNOWFLAKE_TEST_USER"), + "reuse_connections": self.reuse_connections, + **auth_params, + } + + @pytest.fixture(scope="class") + def auth_params(self): + + if self.auth_method == "user_password": + yield {"password": os.getenv("SNOWFLAKE_TEST_PASSWORD")} + + elif self.auth_method == "key_pair": + """ + This connection method uses key pair auth. + Follow the instructions here to setup key pair authentication for your test user: + https://docs.snowflake.com/en/user-guide/key-pair-auth + """ + yield { + "private_key": os.getenv("SNOWFLAKE_TEST_PRIVATE_KEY"), + "private_key_passphrase": os.getenv("SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE"), + } + + else: + raise ValueError( + f"`auth_method` must be one of `user_password` or `key_pair`, received: {self.auth_method}" + ) + + def test_scenario(self, project): + run_dbt(["run"]) + + +class TestUserPasswordAuth(Scenario): + auth_method = "user_password" + project_size = 1_000 + reuse_connections = False + + +class TestUserPasswordAuthReuseConnections(Scenario): + auth_method = "user_password" + project_size = 1_000 + reuse_connections = True + + +class TestKeyPairAuth(Scenario): + auth_method = "key_pair" + project_size = 1_000 + reuse_connections = False + + +class TestKeyPairAuthReuseConnections(Scenario): + auth_method = "key_pair" + project_size = 1_000 + reuse_connections = True diff --git a/dbt-snowflake/tests/unit/__init__.py b/dbt-snowflake/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt-snowflake/tests/unit/mock_adapter.py b/dbt-snowflake/tests/unit/mock_adapter.py new file mode 100644 index 000000000..93394bedc --- /dev/null +++ b/dbt-snowflake/tests/unit/mock_adapter.py @@ -0,0 +1,84 @@ +from contextlib import contextmanager +from unittest import mock + +from dbt.adapters.base import BaseAdapter + + +def adapter_factory(): + class MockAdapter(BaseAdapter): + ConnectionManager = mock.MagicMock(TYPE="mock") + responder = mock.MagicMock() + # some convenient defaults + responder.quote.side_effect = lambda identifier: '"{}"'.format(identifier) + responder.date_function.side_effect = lambda: "unitdate()" + responder.is_cancelable.side_effect = lambda: False + + @contextmanager + def exception_handler(self, *args, **kwargs): + self.responder.exception_handler(*args, **kwargs) + yield + + def execute(self, *args, **kwargs): + return self.responder.execute(*args, **kwargs) + + def drop_relation(self, *args, **kwargs): + return self.responder.drop_relation(*args, **kwargs) + + def truncate_relation(self, *args, **kwargs): + return self.responder.truncate_relation(*args, **kwargs) + + def rename_relation(self, *args, **kwargs): + return self.responder.rename_relation(*args, **kwargs) + + def get_columns_in_relation(self, *args, **kwargs): + return self.responder.get_columns_in_relation(*args, **kwargs) + + def get_catalog_for_single_relation(self, *args, **kwargs): + return self.responder.get_catalog_for_single_relation(*args, **kwargs) + + def expand_column_types(self, *args, **kwargs): + return self.responder.expand_column_types(*args, **kwargs) + + def list_relations_without_caching(self, *args, **kwargs): + return self.responder.list_relations_without_caching(*args, **kwargs) + + def create_schema(self, *args, **kwargs): + return self.responder.create_schema(*args, **kwargs) + + def drop_schema(self, *args, **kwargs): + return self.responder.drop_schema(*args, **kwargs) + + @classmethod + def quote(cls, identifier): + return cls.responder.quote(identifier) + + def convert_text_type(self, *args, **kwargs): + return self.responder.convert_text_type(*args, **kwargs) + + def convert_number_type(self, *args, **kwargs): + return self.responder.convert_number_type(*args, **kwargs) + + def convert_boolean_type(self, *args, **kwargs): + return self.responder.convert_boolean_type(*args, **kwargs) + + def convert_datetime_type(self, *args, **kwargs): + return self.responder.convert_datetime_type(*args, **kwargs) + + def convert_date_type(self, *args, **kwargs): + return self.responder.convert_date_type(*args, **kwargs) + + def convert_time_type(self, *args, **kwargs): + return self.responder.convert_time_type(*args, **kwargs) + + def list_schemas(self, *args, **kwargs): + return self.responder.list_schemas(*args, **kwargs) + + @classmethod + def date_function(cls): + return cls.responder.date_function() + + @classmethod + def is_cancelable(cls): + return cls.responder.is_cancelable() + + return MockAdapter diff --git a/dbt-snowflake/tests/unit/test_adapter_telemetry.py b/dbt-snowflake/tests/unit/test_adapter_telemetry.py new file mode 100644 index 000000000..498676b77 --- /dev/null +++ b/dbt-snowflake/tests/unit/test_adapter_telemetry.py @@ -0,0 +1,27 @@ +from unittest import mock + +import dbt.adapters.snowflake.__version__ + +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.base.relation import AdapterTrackingRelationInfo + + +def test_telemetry_with_snowflake_details(): + mock_model_config = mock.MagicMock() + mock_model_config._extra = mock.MagicMock() + mock_model_config._extra = { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + res = SnowflakeAdapter.get_adapter_run_info(mock_model_config) + + assert res.adapter_name == "snowflake" + assert res.base_adapter_version == dbt.adapters.__about__.version + assert res.adapter_version == dbt.adapters.snowflake.__version__.version + assert res.model_adapter_details == { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + assert type(res) is AdapterTrackingRelationInfo diff --git a/dbt-snowflake/tests/unit/test_connections.py b/dbt-snowflake/tests/unit/test_connections.py new file mode 100644 index 000000000..fb9c57615 --- /dev/null +++ b/dbt-snowflake/tests/unit/test_connections.py @@ -0,0 +1,69 @@ +import os +import pytest +from importlib import reload +from unittest.mock import Mock, patch +import multiprocessing +from dbt.adapters.exceptions.connection import FailedToConnectError +import dbt.adapters.snowflake.connections as connections +import dbt.adapters.events.logging + + +def test_connections_sets_logs_in_response_to_env_var(monkeypatch): + """Test that setting the DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING environment variable happens on import""" + log_mock = Mock() + monkeypatch.setattr(dbt.adapters.events.logging, "AdapterLogger", Mock(return_value=log_mock)) + monkeypatch.setattr(os, "environ", {"DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING": "true"}) + reload(connections) + + assert log_mock.debug.call_count == 3 + assert log_mock.set_adapter_dependency_log_level.call_count == 3 + + +def test_connections_does_not_set_logs_in_response_to_env_var(monkeypatch): + log_mock = Mock() + monkeypatch.setattr(dbt.adapters.events.logging, "AdapterLogger", Mock(return_value=log_mock)) + reload(connections) + + assert log_mock.debug.call_count == 0 + assert log_mock.set_adapter_dependency_log_level.call_count == 0 + + +def test_connnections_credentials_replaces_underscores_with_hyphens(): + credentials = { + "account": "account_id_with_underscores", + "user": "user", + "password": "password", + "database": "database", + "warehouse": "warehouse", + "schema": "schema", + } + creds = connections.SnowflakeCredentials(**credentials) + assert creds.account == "account-id-with-underscores" + + +def test_snowflake_oauth_expired_token_raises_error(): + credentials = { + "account": "test_account", + "user": "test_user", + "authenticator": "oauth", + "token": "expired_or_incorrect_token", + "database": "database", + "schema": "schema", + } + + mp_context = multiprocessing.get_context("spawn") + mock_credentials = connections.SnowflakeCredentials(**credentials) + + with patch.object( + connections.SnowflakeConnectionManager, + "open", + side_effect=FailedToConnectError( + "This error occurs when authentication has expired. " + "Please reauth with your auth provider." + ), + ): + + adapter = connections.SnowflakeConnectionManager(mock_credentials, mp_context) + + with pytest.raises(FailedToConnectError): + adapter.open() diff --git a/dbt-snowflake/tests/unit/test_iceberg_location.py b/dbt-snowflake/tests/unit/test_iceberg_location.py new file mode 100644 index 000000000..dca82b47e --- /dev/null +++ b/dbt-snowflake/tests/unit/test_iceberg_location.py @@ -0,0 +1,79 @@ +import pytest +from dbt.adapters.snowflake.relation import SnowflakeRelation + + +@pytest.fixture +def iceberg_config() -> dict: + """Fixture providing standard Iceberg configuration.""" + return { + "schema": "my_schema", + "identifier": "my_table", + "external_volume": "s3_iceberg_snow", + "base_location_root": "root_path", + "base_location_subpath": "subpath", + } + + +def get_actual_base_location(config: dict[str, str]) -> str: + """Get the actual base location from the configuration by parsing the DDL predicates.""" + + relation = SnowflakeRelation.create( + schema=config["schema"], + identifier=config["identifier"], + ) + + actual_ddl_predicates = relation.get_iceberg_ddl_options(config).strip() + actual_base_location = actual_ddl_predicates.split("base_location = ")[1] + + return actual_base_location + + +def test_iceberg_path_and_subpath(iceberg_config: dict[str, str]): + """Test when base_location_root and base_location_subpath are provided""" + expected_base_location = ( + f"'{iceberg_config['base_location_root']}/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}/" + f"{iceberg_config['base_location_subpath']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_only_subpath(iceberg_config: dict[str, str]): + """Test when only base_location_subpath is provided""" + del iceberg_config["base_location_root"] + + expected_base_location = ( + f"'_dbt/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}/" + f"{iceberg_config['base_location_subpath']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_only_path(iceberg_config: dict[str, str]): + """Test when only base_location_root is provided""" + del iceberg_config["base_location_subpath"] + + expected_base_location = ( + f"'{iceberg_config['base_location_root']}/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_no_path(iceberg_config: dict[str, str]): + """Test when no base_location_root or is base_location_subpath provided""" + del iceberg_config["base_location_root"] + del iceberg_config["base_location_subpath"] + + expected_base_location = ( + f"'_dbt/" f"{iceberg_config['schema']}/" f"{iceberg_config['identifier']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location diff --git a/dbt-snowflake/tests/unit/test_private_keys.py b/dbt-snowflake/tests/unit/test_private_keys.py new file mode 100644 index 000000000..59b8522d2 --- /dev/null +++ b/dbt-snowflake/tests/unit/test_private_keys.py @@ -0,0 +1,61 @@ +import os +import tempfile +from typing import Generator + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import pytest + +from dbt.adapters.snowflake.auth import private_key_from_file, private_key_from_string + + +PASSPHRASE = "password1234" + + +def serialize(private_key: rsa.RSAPrivateKey) -> bytes: + return private_key.private_bytes( + serialization.Encoding.DER, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + + +@pytest.fixture(scope="session") +def private_key() -> rsa.RSAPrivateKey: + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +@pytest.fixture(scope="session") +def private_key_string(private_key) -> str: + private_key_bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.BestAvailableEncryption(PASSPHRASE.encode()), + ) + return private_key_bytes.decode("utf-8") + + +@pytest.fixture(scope="session") +def private_key_file(private_key) -> Generator[str, None, None]: + private_key_bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.BestAvailableEncryption(PASSPHRASE.encode()), + ) + file = tempfile.NamedTemporaryFile() + file.write(private_key_bytes) + file.seek(0) + yield file.name + file.close() + + +def test_private_key_from_string_pem(private_key_string, private_key): + assert isinstance(private_key_string, str) + calculated_private_key = private_key_from_string(private_key_string, PASSPHRASE) + assert serialize(calculated_private_key) == serialize(private_key) + + +def test_private_key_from_file(private_key_file, private_key): + assert os.path.exists(private_key_file) + calculated_private_key = private_key_from_file(private_key_file, PASSPHRASE) + assert serialize(calculated_private_key) == serialize(private_key) diff --git a/dbt-snowflake/tests/unit/test_relation_as_case_sensitive.py b/dbt-snowflake/tests/unit/test_relation_as_case_sensitive.py new file mode 100644 index 000000000..f362d66b3 --- /dev/null +++ b/dbt-snowflake/tests/unit/test_relation_as_case_sensitive.py @@ -0,0 +1,19 @@ +from dbt.adapters.snowflake.relation import SnowflakeRelation +from dbt.adapters.snowflake.relation_configs import SnowflakeQuotePolicy + + +def test_relation_as_case_sensitive_quoting_true(): + relation = SnowflakeRelation.create( + database="My_Db", + schema="My_ScHeMa", + identifier="My_TaBlE", + quote_policy=SnowflakeQuotePolicy(database=False, schema=True, identifier=False), + ) + + case_sensitive_relation = relation.as_case_sensitive() + case_sensitive_relation.render_limited() + + assert case_sensitive_relation.database == "MY_DB" + assert case_sensitive_relation.schema == "My_ScHeMa" + assert case_sensitive_relation.identifier == "MY_TABLE" + assert case_sensitive_relation.render() == 'MY_DB."My_ScHeMa".MY_TABLE' diff --git a/dbt-snowflake/tests/unit/test_renamed_relations.py b/dbt-snowflake/tests/unit/test_renamed_relations.py new file mode 100644 index 000000000..315d5f343 --- /dev/null +++ b/dbt-snowflake/tests/unit/test_renamed_relations.py @@ -0,0 +1,17 @@ +from dbt.adapters.snowflake.relation import SnowflakeRelation +from dbt.adapters.snowflake.relation_configs import SnowflakeRelationType + + +def test_renameable_relation(): + relation = SnowflakeRelation.create( + database="my_db", + schema="my_schema", + identifier="my_table", + type=SnowflakeRelationType.Table, + ) + assert relation.renameable_relations == frozenset( + { + SnowflakeRelationType.Table, + SnowflakeRelationType.View, + } + ) diff --git a/dbt-snowflake/tests/unit/test_snowflake_adapter.py b/dbt-snowflake/tests/unit/test_snowflake_adapter.py new file mode 100644 index 000000000..18a5310dc --- /dev/null +++ b/dbt-snowflake/tests/unit/test_snowflake_adapter.py @@ -0,0 +1,1009 @@ +import agate +import re +import unittest +from multiprocessing import get_context +from contextlib import contextmanager +from unittest import mock + +from dbt.adapters.snowflake import SnowflakeAdapter +from dbt.adapters.snowflake import Plugin as SnowflakePlugin +from dbt.adapters.snowflake.column import SnowflakeColumn +from dbt.adapters.snowflake.connections import SnowflakeCredentials +from dbt.contracts.files import FileHash +from dbt.context.query_header import generate_query_header_context +from dbt.context.providers import generate_runtime_macro_context +from dbt.contracts.graph.manifest import ManifestStateCheck +from dbt_common.clients import agate_helper +from snowflake import connector as snowflake_connector + +from .utils import ( + config_from_parts_or_dicts, + inject_adapter, + mock_connection, + TestAdapterConversions, + load_internal_manifest_macros, +) + + +class TestSnowflakeAdapter(unittest.TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "snowflake", + "account": "test_account", + "user": "test_user", + "database": "test_database", + "warehouse": "test_warehouse", + "schema": "public", + }, + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "query-comment": "dbt", + "config-version": 2, + } + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self.assertEqual(self.config.query_comment.comment, "dbt") + self.assertEqual(self.config.query_comment.append, False) + + self.handle = mock.MagicMock(spec=snowflake_connector.SnowflakeConnection) + self.cursor = self.handle.cursor.return_value + self.mock_execute = self.cursor.execute + self.mock_execute.return_value = mock.MagicMock(sfqid="42") + self.patcher = mock.patch("dbt.adapters.snowflake.connections.snowflake.connector.connect") + self.snowflake = self.patcher.start() + self.snowflake.connect.cursor.return_value = mock.MagicMock(sfqid="42") + + # Create the Manifest.state_check patcher + @mock.patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") + def _mock_state_check(self): + all_projects = self.all_projects + return ManifestStateCheck( + vars_hash=FileHash.from_contents("vars"), + project_hashes={name: FileHash.from_contents(name) for name in all_projects}, + profile_hash=FileHash.from_contents("profile"), + ) + + self.load_state_check = mock.patch( + "dbt.parser.manifest.ManifestLoader.build_manifest_state_check" + ) + self.mock_state_check = self.load_state_check.start() + self.mock_state_check.side_effect = _mock_state_check + + self.snowflake.return_value = self.handle + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + self.adapter.set_macro_resolver(load_internal_manifest_macros(self.config)) + self.adapter.set_macro_context_generator(generate_runtime_macro_context) + self.adapter.connections.set_query_header( + generate_query_header_context(self.config, self.adapter.get_macro_resolver()) + ) + + self.qh_patch = mock.patch.object(self.adapter.connections.query_header, "add") + self.mock_query_header_add = self.qh_patch.start() + self.mock_query_header_add.side_effect = lambda q: "/* dbt */\n{}".format(q) + self.adapter.acquire_connection() + inject_adapter(self.adapter, SnowflakePlugin) + + def tearDown(self): + # we want a unique self.handle every time. + self.adapter.cleanup_connections() + self.qh_patch.stop() + self.patcher.stop() + self.load_state_check.stop() + + def test_quoting_on_drop_schema(self): + relation = SnowflakeAdapter.Relation.create( + database="test_database", + schema="test_schema", + quote_policy=self.adapter.config.quoting, + ) + self.adapter.drop_schema(relation) + + self.mock_execute.assert_has_calls( + [ + mock.call( + '/* dbt */\ndrop schema if exists test_database."test_schema" cascade', None + ) + ] + ) + + def test_quoting_on_drop(self): + relation = self.adapter.Relation.create( + database="test_database", + schema="test_schema", + identifier="test_table", + type="table", + quote_policy=self.adapter.config.quoting, + ) + self.adapter.drop_relation(relation) + + self.mock_execute.assert_has_calls( + [ + mock.call( + '/* dbt */\ndrop table if exists test_database."test_schema".test_table cascade', + None, + ) + ] + ) + + def test_quoting_on_truncate(self): + relation = self.adapter.Relation.create( + database="test_database", + schema="test_schema", + identifier="test_table", + type="table", + quote_policy=self.adapter.config.quoting, + ) + self.adapter.truncate_relation(relation) + + # no query comment because wrapped in begin; + commit; for explicit DML + self.mock_execute.assert_has_calls( + [ + mock.call("/* dbt */\nBEGIN", None), + mock.call( + '/* dbt */\ntruncate table test_database."test_schema".test_table\n ;', None + ), + mock.call("/* dbt */\nCOMMIT", None), + ] + ) + + def test_quoting_on_rename(self): + from_relation = self.adapter.Relation.create( + database="test_database", + schema="test_schema", + identifier="table_a", + type="table", + quote_policy=self.adapter.config.quoting, + ) + to_relation = self.adapter.Relation.create( + database="test_database", + schema="test_schema", + identifier="table_b", + type="table", + quote_policy=self.adapter.config.quoting, + ) + + self.adapter.rename_relation(from_relation=from_relation, to_relation=to_relation) + self.mock_execute.assert_has_calls( + [ + mock.call( + '/* dbt */\nalter table test_database."test_schema".table_a rename to test_database."test_schema".table_b', + None, + ) + ] + ) + + @contextmanager + def current_warehouse(self, response): + # there is probably some elegant way built into mock.patch to do this + fetchall_return = self.cursor.fetchall.return_value + execute_side_effect = self.mock_execute.side_effect + + def execute_effect(sql, *args, **kwargs): + if sql == "/* dbt */\nselect current_warehouse() as warehouse": + self.cursor.description = [["name"]] + self.cursor.fetchall.return_value = [[response]] + else: + self.cursor.description = None + self.cursor.fetchall.return_value = fetchall_return + return self.mock_execute.return_value + + self.mock_execute.side_effect = execute_effect + try: + yield + finally: + self.cursor.fetchall.return_value = fetchall_return + self.mock_execute.side_effect = execute_side_effect + + def _strip_transactions(self): + result = [] + for call_args in self.mock_execute.call_args_list: + args, kwargs = tuple(call_args) + is_transactional = ( + len(kwargs) == 0 + and len(args) == 2 + and args[1] is None + and args[0] in {"BEGIN", "COMMIT"} + ) + if not is_transactional: + result.append(call_args) + return result + + def test_pre_post_hooks_warehouse(self): + with self.current_warehouse("warehouse"): + config = {"snowflake_warehouse": "other_warehouse"} + result = self.adapter.pre_model_hook(config) + self.assertIsNotNone(result) + calls = [ + mock.call("/* dbt */\nselect current_warehouse() as warehouse", None), + mock.call("/* dbt */\nuse warehouse other_warehouse", None), + ] + self.mock_execute.assert_has_calls(calls) + self.adapter.post_model_hook(config, result) + calls.append(mock.call("/* dbt */\nuse warehouse warehouse", None)) + self.mock_execute.assert_has_calls(calls) + + def test_pre_post_hooks_no_warehouse(self): + with self.current_warehouse("warehouse"): + config = {} + result = self.adapter.pre_model_hook(config) + self.assertIsNone(result) + self.mock_execute.assert_not_called() + self.adapter.post_model_hook(config, result) + self.mock_execute.assert_not_called() + + def test_cancel_open_connections_empty(self): + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_master(self): + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections[key] = mock_connection("master") + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_single(self): + master = mock_connection("master") + model = mock_connection("model") + model.handle.session_id = 42 + + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections.update( + { + key: master, + 1: model, + } + ) + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + query_result = mock.MagicMock() + add_query.return_value = (None, query_result) + + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) + + add_query.assert_called_once_with("select system$cancel_all_queries(42)") + + def test_client_session_keep_alive_false_by_default(self): + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ), + ] + ) + + def test_client_session_keep_alive_true(self): + self.config.credentials = self.config.credentials.replace( + client_session_keep_alive=True, + # this gets defaulted via `__post_init__` when `client_session_keep_alive` comes in as `False` + # then when `replace` is called, `__post_init__` cannot set it back to `None` since it cannot + # tell the difference between set by user and set by `__post_init__` + reuse_connections=None, + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=True, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=None, + ) + ] + ) + + def test_client_has_query_tag(self): + self.config.credentials = self.config.credentials.replace(query_tag="test_query_tag") + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + reuse_connections=True, + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={"QUERY_TAG": "test_query_tag"}, + ) + ] + ) + + expected_connection_info = [ + (k, v) for (k, v) in self.config.credentials.connection_info() if k == "query_tag" + ] + self.assertEqual([("query_tag", "test_query_tag")], expected_connection_info) + + def test_user_pass_authentication(self): + self.config.credentials = self.config.credentials.replace( + password="test_password", + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + password="test_password", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + def test_authenticator_user_pass_authentication(self): + self.config.credentials = self.config.credentials.replace( + password="test_password", + authenticator="test_sso_url", + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + password="test_password", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + authenticator="test_sso_url", + private_key=None, + application="dbt", + client_request_mfa_token=True, + client_store_temporary_credential=True, + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + def test_authenticator_externalbrowser_authentication(self): + self.config.credentials = self.config.credentials.replace(authenticator="externalbrowser") + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + authenticator="externalbrowser", + private_key=None, + application="dbt", + client_request_mfa_token=True, + client_store_temporary_credential=True, + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + def test_authenticator_oauth_authentication(self): + self.config.credentials = self.config.credentials.replace( + authenticator="oauth", + token="my-oauth-token", + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + authenticator="oauth", + token="my-oauth-token", + private_key=None, + application="dbt", + client_request_mfa_token=True, + client_store_temporary_credential=True, + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + @mock.patch( + "dbt.adapters.snowflake.SnowflakeCredentials._get_private_key", return_value="test_key" + ) + def test_authenticator_private_key_authentication(self, mock_get_private_key): + self.config.credentials = self.config.credentials.replace( + private_key_path="/tmp/test_key.p8", + private_key_passphrase="p@ssphr@se", + ) + + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key="test_key", + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + @mock.patch( + "dbt.adapters.snowflake.SnowflakeCredentials._get_private_key", return_value="test_key" + ) + def test_authenticator_private_key_authentication_no_passphrase(self, mock_get_private_key): + self.config.credentials = self.config.credentials.replace( + private_key_path="/tmp/test_key.p8", + private_key_passphrase=None, + ) + + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key="test_key", + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + def test_authenticator_jwt_authentication(self): + self.config.credentials = self.config.credentials.replace( + authenticator="jwt", token="my-jwt-token", user=None + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + warehouse="test_warehouse", + authenticator="oauth", + token="my-jwt-token", + private_key=None, + application="dbt", + client_request_mfa_token=True, + client_store_temporary_credential=True, + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + def test_query_tag(self): + self.config.credentials = self.config.credentials.replace( + password="test_password", query_tag="test_query_tag" + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + password="test_password", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={"QUERY_TAG": "test_query_tag"}, + reuse_connections=True, + ) + ] + ) + + def test_reuse_connections_with_keep_alive(self): + self.config.credentials = self.config.credentials.replace( + reuse_connections=True, client_session_keep_alive=True + ) + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=True, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key=None, + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + @mock.patch( + "dbt.adapters.snowflake.SnowflakeCredentials._get_private_key", return_value="test_key" + ) + def test_authenticator_private_key_string_authentication(self, mock_get_private_key): + self.config.credentials = self.config.credentials.replace( + private_key="dGVzdF9rZXk=", + private_key_passphrase="p@ssphr@se", + ) + + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key="test_key", + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + @mock.patch( + "dbt.adapters.snowflake.SnowflakeCredentials._get_private_key", return_value="test_key" + ) + def test_authenticator_private_key_string_authentication_no_passphrase( + self, mock_get_private_key + ): + self.config.credentials = self.config.credentials.replace( + private_key="dGVzdF9rZXk=", + private_key_passphrase=None, + ) + + self.adapter = SnowflakeAdapter(self.config, get_context("spawn")) + conn = self.adapter.connections.set_connection_name(name="new_connection_with_new_config") + + self.snowflake.assert_not_called() + conn.handle + self.snowflake.assert_has_calls( + [ + mock.call( + account="test-account", + autocommit=True, + client_session_keep_alive=False, + database="test_database", + role=None, + schema="public", + user="test_user", + warehouse="test_warehouse", + private_key="test_key", + application="dbt", + insecure_mode=False, + session_parameters={}, + reuse_connections=True, + ) + ] + ) + + +class TestSnowflakeAdapterConversions(TestAdapterConversions): + def test_convert_text_type(self): + rows = [ + ["", "a1", "stringval1"], + ["", "a2", "stringvalasdfasdfasdfa"], + ["", "a3", "stringval3"], + ] + agate_table = self._make_table_of(rows, agate.Text) + expected = ["text", "text", "text"] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_text_type(agate_table, col_idx) == expect + + def test_convert_number_type(self): + rows = [ + ["", "23.98", "-1"], + ["", "12.78", "-2"], + ["", "79.41", "-3"], + ] + agate_table = self._make_table_of(rows, agate.Number) + expected = ["integer", "float8", "integer"] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_number_type(agate_table, col_idx) == expect + + def test_convert_boolean_type(self): + rows = [ + ["", "false", "true"], + ["", "false", "false"], + ["", "false", "true"], + ] + agate_table = self._make_table_of(rows, agate.Boolean) + expected = ["boolean", "boolean", "boolean"] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_boolean_type(agate_table, col_idx) == expect + + def test_convert_datetime_type(self): + rows = [ + ["", "20190101T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190102T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190103T01:01:01Z", "2019-01-01 01:01:01"], + ] + agate_table = self._make_table_of( + rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime] + ) + expected = [ + "timestamp without time zone", + "timestamp without time zone", + "timestamp without time zone", + ] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_datetime_type(agate_table, col_idx) == expect + + def test_convert_date_type(self): + rows = [ + ["", "2019-01-01", "2019-01-04"], + ["", "2019-01-02", "2019-01-04"], + ["", "2019-01-03", "2019-01-04"], + ] + agate_table = self._make_table_of(rows, agate.Date) + expected = ["date", "date", "date"] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_date_type(agate_table, col_idx) == expect + + def test_convert_time_type(self): + # dbt's default type testers actually don't have a TimeDelta at all. + agate.TimeDelta + rows = [ + ["", "120s", "10s"], + ["", "3m", "11s"], + ["", "1h", "12s"], + ] + agate_table = self._make_table_of(rows, agate.TimeDelta) + expected = ["time", "time", "time"] + for col_idx, expect in enumerate(expected): + assert SnowflakeAdapter.convert_time_type(agate_table, col_idx) == expect + + +class TestSnowflakeColumn(unittest.TestCase): + def test_text_from_description(self): + col = SnowflakeColumn.from_description("my_col", "TEXT") + assert col.column == "my_col" + assert col.dtype == "TEXT" + assert col.char_size is None + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is False + assert col.is_number() is False + assert col.is_numeric() is False + assert col.is_string() is True + assert col.is_integer() is False + assert col.string_size() == 16777216 + + col = SnowflakeColumn.from_description("my_col", "VARCHAR") + assert col.column == "my_col" + assert col.dtype == "VARCHAR" + assert col.char_size is None + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is False + assert col.is_number() is False + assert col.is_numeric() is False + assert col.is_string() is True + assert col.is_integer() is False + assert col.string_size() == 16777216 + + def test_sized_varchar_from_description(self): + col = SnowflakeColumn.from_description("my_col", "VARCHAR(256)") + assert col.column == "my_col" + assert col.dtype == "VARCHAR" + assert col.char_size == 256 + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is False + assert col.is_number() is False + assert col.is_numeric() is False + assert col.is_string() is True + assert col.is_integer() is False + assert col.string_size() == 256 + + def test_sized_decimal_from_description(self): + col = SnowflakeColumn.from_description("my_col", "DECIMAL(1, 0)") + assert col.column == "my_col" + assert col.dtype == "DECIMAL" + assert col.char_size is None + assert col.numeric_precision == 1 + assert col.numeric_scale == 0 + assert col.is_float() is False + assert col.is_number() is True + assert col.is_numeric() is True + assert col.is_string() is False + assert col.is_integer() is False + + def test_float_from_description(self): + col = SnowflakeColumn.from_description("my_col", "FLOAT8") + assert col.column == "my_col" + assert col.dtype == "FLOAT8" + assert col.char_size is None + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is True + assert col.is_number() is True + assert col.is_numeric() is False + assert col.is_string() is False + assert col.is_integer() is False + + def test_vector_from_description(self): + col = SnowflakeColumn.from_description("my_col", "VECTOR(FLOAT, 768)") + assert col.column == "my_col" + assert col.dtype == "VECTOR(FLOAT, 768)" + assert col.char_size is None + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is False + assert col.is_number() is False + assert col.is_numeric() is False + assert col.is_string() is False + assert col.is_integer() is False + + +class SnowflakeConnectionsTest(unittest.TestCase): + def test_comment_stripping_regex(self): + pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|--[^\r\n]*$)" + comment1 = "-- just comment" + comment2 = "/* just comment */" + query1 = "select 1; -- comment" + query2 = "select 1; /* comment */" + query3 = "select 1; -- comment\nselect 2; /* comment */ " + query4 = "select \n1; -- comment\nselect \n2; /* comment */ " + query5 = "select 1; -- comment \nselect 2; -- comment \nselect 3; -- comment" + + stripped_comment1 = re.sub(re.compile(pattern, re.MULTILINE), "", comment1).strip() + + stripped_comment2 = re.sub(re.compile(pattern, re.MULTILINE), "", comment2).strip() + + stripped_query1 = re.sub(re.compile(pattern, re.MULTILINE), "", query1).strip() + + stripped_query2 = re.sub(re.compile(pattern, re.MULTILINE), "", query2).strip() + + stripped_query3 = re.sub(re.compile(pattern, re.MULTILINE), "", query3).strip() + + stripped_query4 = re.sub(re.compile(pattern, re.MULTILINE), "", query4).strip() + + stripped_query5 = re.sub(re.compile(pattern, re.MULTILINE), "", query5).strip() + + expected_query_3 = "select 1; \nselect 2;" + expected_query_4 = "select \n1; \nselect \n2;" + expected_query_5 = "select 1; \nselect 2; \nselect 3;" + + self.assertEqual("", stripped_comment1) + self.assertEqual("", stripped_comment2) + self.assertEqual("select 1;", stripped_query1) + self.assertEqual("select 1;", stripped_query2) + self.assertEqual(expected_query_3, stripped_query3) + self.assertEqual(expected_query_4, stripped_query4) + self.assertEqual(expected_query_5, stripped_query5) + + +class TestSnowflakeAdapterCredentials(unittest.TestCase): + unencrypted_private_key = ( + b"0\x82\x01T\x02\x01\x000\r\x06\t*\x86H\x86\xf7\r\x01\x01\x01" + b"\x05\x00\x04\x82\x01>0\x82\x01:\x02\x01\x00\x02A\x00\xd9." + b"\x15\xc8\xce\xfa\x1c\x9a\xe8/|5lf\xb8\xd9\x13\xc5I\x16i \x9f" + b"'rO\xb1RkD(\n\xff\x84v\xbaS\x8d\xb46\xf8\x85w\x81\xe2\xc5cy" + b"\xf1\xb6\xa9i]F\xfc\x04e`\xfbw;\x91\xf5\xcf\x02\x03\x01\x00" + b"\x01\x02A\x00\x81\x84\xc6a\x17ny\x98\xb8WyO\xb2\xf2\x1f\xd2" + b"\xf5\xc3v.\xf3K\r\x1fM@\xd1\x93A}H\x13\r\xa7\xd4\n,7L\x14?" + b"\xff\xe2\xf3\xac\x93\xbb\xdf\xc3\xe5\xea\xf1AG\xc0~\xa2\x9a6" + b"6\xeb\x11S\xe1\x02!\x00\xf3\x1d\xf1\xcc\xecj\xaf}\x01\xd4" + b"\xee\x84\x03(Qx9\x9f\xedH\xf1\x016r\xbaE{Uk\x9d,\x13\x02!" + b"\x00\xe4\xb0I\x8b\x8bU\xe5\x9a\x93V\x9f\xa8Ui\x9cQ\xd7\x12" + b"\x866g\xdf[{y\xe8\xc1j\xda\xc2\xa9' + b"\x9d\x94)_\xe6\x9c\xf1FF\x03Y\x81" + ) + unencrypted_private_key_encoded = ( + "MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEA2S4VyM76HJroL3w1bGa42" + "RPFSRZpIJ8nck+xUmtEKAr/hHa6U420NviFd4HixWN58bapaV1G/ARlYPt3O5H1zwIDAQ" + "ABAkEAgYTGYRdueZi4V3lPsvIf0vXDdi7zSw0fTUDRk0F9SBMNp9QKLDdMFD//4vOsk7v" + "fw+Xq8UFHwH6imjY26xFT4QIhAPMd8czsaq99AdTuhAMoUXg5n+1I8QE2crpFe1VrnSwT" + "AiEA5LBJi4tV5ZqTVp+oVWmcUdcShjZnPE1LpEo1sMb3ztUCIHdwPavkdiFS88RtjZOTi" + "jrbbJOBo01qN4EFw6ranLPbAiB3r1deQ9b5qnNwA3D6+qHIJzJXsYNICQAMhJYi5Y7t3Q" + "IgHVwCvPP/FyI6Pt9be3nowWrawqmdlClf5pzxRkYDWYE=" + ) + encrypted_private_key_encoded = ( + "MIIBvTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIIxdFbtlFbgkCAggAMAwGC" + "CqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBBswzZouyovzUADDQcLUEwgBIIBYAiB+rnGhm" + "PwKZYOFdyEvkXFFu2aRfqotYHy/qlfVdU4BfNHwBlAlgUOPMN2HJ9KwyiNdBKoQ1Z4KXI" + "G4AU74QZsVSL+miFf65qqWKLwckL45Y3WMUH1K0YpdO0W+aznjH9msWYuM/zCpQS2rvVX" + "a5rXpA5praB5nn6kRlTwrQ8DN0ZKOKBX6ojhSE/6TmQtx3d+tmly8ZpTkG5HVTuBMCtDg" + "Po6mAEvvb4T/dnx9MtUz0d5AgNuVOS5+OI32kX9XVupEGkvdY8iHbx7+skbKFVdHMayBL" + "0dy5knySv+YGi/T6oM0uApPPk4aT493MNzT8544Wmi/NbNkDx+6XQiTuPZ4OsL0iF3KsX" + "xfTC4tDGGYn4yW8bnSz3K+lkXA9vyie37nW6ncu+aizT9TgD1q6jFm9u/1G61/Z96oHgz" + "pVghkP6s3l23U/7qM2PC8CEu18nUDYEhrv6lOwr8EABHV0s=" + ) + encrypted_private_key_passphrase = "insecure" + + def test_private_key_string(self): + creds = SnowflakeCredentials( + account="test-account", + user="test_user", + database="test_database", + schema="public", + private_key=self.unencrypted_private_key_encoded, + ) + self.assertEqual(creds.auth_args()["private_key"], self.unencrypted_private_key) + + def test_private_key_string_encrypted(self): + creds = SnowflakeCredentials( + account="test-account", + user="test_user", + database="test_database", + schema="public", + private_key=self.encrypted_private_key_encoded, + private_key_passphrase=self.encrypted_private_key_passphrase, + ) + self.assertEqual(creds.auth_args()["private_key"], self.unencrypted_private_key) + + def test_malformed_private_key_string(self): + creds = SnowflakeCredentials( + account="test-account", + user="test_user", + database="test_database", + schema="public", + private_key="dGVzdF9rZXk=", + ) + self.assertRaises(ValueError, creds.auth_args) + + def test_invalid_private_key_string(self): + creds = SnowflakeCredentials( + account="test-account", + user="test_user", + database="test_database", + schema="public", + private_key="invalid[base64]=", + ) + self.assertRaises(ValueError, creds.auth_args) + + def test_invalid_private_key_path(self): + creds = SnowflakeCredentials( + account="test-account", + user="test_user", + database="test_database", + schema="public", + private_key_path="/tmp/does/not/exist.p8", + ) + self.assertRaises(FileNotFoundError, creds.auth_args) diff --git a/dbt-snowflake/tests/unit/utils.py b/dbt-snowflake/tests/unit/utils.py new file mode 100644 index 000000000..c7ec66bf4 --- /dev/null +++ b/dbt-snowflake/tests/unit/utils.py @@ -0,0 +1,271 @@ +"""Unit test utility functions. +Note that all imports should be inside the functions to avoid import/mocking +issues. +""" + +import string +import os +from unittest import mock +from unittest import TestCase + +import agate +import pytest +from dbt_common.dataclass_schema import ValidationError +from dbt.config.project import PartialProject + + +def normalize(path): + """On windows, neither is enough on its own: + >>> normcase('C:\\documents/ALL CAPS/subdir\\..') + 'c:\\documents\\all caps\\subdir\\..' + >>> normpath('C:\\documents/ALL CAPS/subdir\\..') + 'C:\\documents\\ALL CAPS' + >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) + 'c:\\documents\\all caps' + """ + return os.path.normcase(os.path.normpath(path)) + + +class Obj: + which = "blah" + single_threaded = False + + +def mock_connection(name, state="open"): + conn = mock.MagicMock() + conn.name = name + conn.state = state + return conn + + +def profile_from_dict(profile, profile_name, cli_vars="{}"): + from dbt.config import Profile + from dbt.config.renderer import ProfileRenderer + from dbt.config.utils import parse_cli_vars + + if not isinstance(cli_vars, dict): + cli_vars = parse_cli_vars(cli_vars) + + renderer = ProfileRenderer(cli_vars) + + # in order to call dbt's internal profile rendering, we need to set the + # flags global. This is a bit of a hack, but it's the best way to do it. + from dbt.flags import set_from_args + from argparse import Namespace + + set_from_args(Namespace(), None) + return Profile.from_raw_profile_info( + profile, + profile_name, + renderer, + ) + + +def project_from_dict(project, profile, packages=None, selectors=None, cli_vars="{}"): + from dbt.config.renderer import DbtProjectYamlRenderer + from dbt.config.utils import parse_cli_vars + + if not isinstance(cli_vars, dict): + cli_vars = parse_cli_vars(cli_vars) + + renderer = DbtProjectYamlRenderer(profile, cli_vars) + + project_root = project.pop("project-root", os.getcwd()) + + partial = PartialProject.from_dicts( + project_root=project_root, + project_dict=project, + packages_dict=packages, + selectors_dict=selectors, + ) + return partial.render(renderer) + + +def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, cli_vars="{}"): + from dbt.config import Project, Profile, RuntimeConfig + from dbt.config.utils import parse_cli_vars + from copy import deepcopy + + if not isinstance(cli_vars, dict): + cli_vars = parse_cli_vars(cli_vars) + + if isinstance(project, Project): + profile_name = project.profile_name + else: + profile_name = project.get("profile") + + if not isinstance(profile, Profile): + profile = profile_from_dict( + deepcopy(profile), + profile_name, + cli_vars, + ) + + if not isinstance(project, Project): + project = project_from_dict( + deepcopy(project), + profile, + packages, + selectors, + cli_vars, + ) + + args = Obj() + args.vars = cli_vars + args.profile_dir = "/dev/null" + return RuntimeConfig.from_parts(project=project, profile=profile, args=args) + + +def inject_plugin(plugin): + from dbt.adapters.factory import FACTORY + + key = plugin.adapter.type() + FACTORY.plugins[key] = plugin + + +def inject_plugin_for(config): + # from dbt.adapters.postgres import Plugin, PostgresAdapter + from dbt.adapters.factory import FACTORY + + FACTORY.load_plugin(config.credentials.type) + adapter = FACTORY.get_adapter(config) + return adapter + + +def inject_adapter(value, plugin): + """Inject the given adapter into the adapter factory, so your hand-crafted + artisanal adapter will be available from get_adapter() as if dbt loaded it. + """ + inject_plugin(plugin) + from dbt.adapters.factory import FACTORY + + key = value.type() + FACTORY.adapters[key] = value + + +def clear_plugin(plugin): + from dbt.adapters.factory import FACTORY + + key = plugin.adapter.type() + FACTORY.plugins.pop(key, None) + FACTORY.adapters.pop(key, None) + + +class ContractTestCase(TestCase): + ContractType = None + + def setUp(self): + self.maxDiff = None + super().setUp() + + def assert_to_dict(self, obj, dct): + self.assertEqual(obj.to_dict(omit_none=True), dct) + + def assert_from_dict(self, obj, dct, cls=None): + if cls is None: + cls = self.ContractType + cls.validate(dct) + self.assertEqual(cls.from_dict(dct), obj) + + def assert_symmetric(self, obj, dct, cls=None): + self.assert_to_dict(obj, dct) + self.assert_from_dict(obj, dct, cls) + + def assert_fails_validation(self, dct, cls=None): + if cls is None: + cls = self.ContractType + + with self.assertRaises(ValidationError): + cls.validate(dct) + cls.from_dict(dct) + + +def compare_dicts(dict1, dict2): + first_set = set(dict1.keys()) + second_set = set(dict2.keys()) + print(f"--- Difference between first and second keys: {first_set.difference(second_set)}") + print(f"--- Difference between second and first keys: {second_set.difference(first_set)}") + common_keys = set(first_set).intersection(set(second_set)) + found_differences = False + for key in common_keys: + if dict1[key] != dict2[key]: + print(f"--- --- first dict: {key}: {str(dict1[key])}") + print(f"--- --- second dict: {key}: {str(dict2[key])}") + found_differences = True + if found_differences: + print("--- Found differences in dictionaries") + else: + print("--- Found no differences in dictionaries") + + +def assert_from_dict(obj, dct, cls=None): + if cls is None: + cls = obj.__class__ + cls.validate(dct) + obj_from_dict = cls.from_dict(dct) + if hasattr(obj, "created_at"): + obj_from_dict.created_at = 1 + obj.created_at = 1 + assert obj_from_dict == obj + + +def assert_to_dict(obj, dct): + obj_to_dict = obj.to_dict(omit_none=True) + if "created_at" in obj_to_dict: + obj_to_dict["created_at"] = 1 + if "created_at" in dct: + dct["created_at"] = 1 + assert obj_to_dict == dct + + +def assert_symmetric(obj, dct, cls=None): + assert_to_dict(obj, dct) + assert_from_dict(obj, dct, cls) + + +def assert_fails_validation(dct, cls): + with pytest.raises(ValidationError): + cls.validate(dct) + cls.from_dict(dct) + + +class TestAdapterConversions(TestCase): + def _get_tester_for(self, column_type): + from dbt_common.clients import agate_helper + + if column_type is agate.TimeDelta: # dbt never makes this! + return agate.TimeDelta() + + for instance in agate_helper.DEFAULT_TYPE_TESTER._possible_types: + if isinstance(instance, column_type): # include child types + return instance + + raise ValueError(f"no tester for {column_type}") + + def _make_table_of(self, rows, column_types): + column_names = list(string.ascii_letters[: len(rows[0])]) + if isinstance(column_types, type): + column_types = [self._get_tester_for(column_types) for _ in column_names] + else: + column_types = [self._get_tester_for(typ) for typ in column_types] + table = agate.Table(rows, column_names=column_names, column_types=column_types) + return table + + +def load_internal_manifest_macros(config, macro_hook=lambda m: None): + from dbt.parser.manifest import ManifestLoader + + return ManifestLoader.load_macros(config, macro_hook) + + +def dict_replace(dct, **kwargs): + dct = dct.copy() + dct.update(kwargs) + return dct + + +def replace_config(n, **kwargs): + return n.replace( + config=n.config.replace(**kwargs), + unrendered_config=dict_replace(n.unrendered_config, **kwargs), + )