Skip to content

Commit

Permalink
Merge pull request #681 from dbt-labs/monorepo/migrate-dbt-snowflake
Browse files Browse the repository at this point in the history
ADAP-1129: Migrate dbt-snowflake into dbt-adapters
  • Loading branch information
mikealfare authored Jan 24, 2025
2 parents b92b722 + 2054f1c commit aa29250
Show file tree
Hide file tree
Showing 196 changed files with 12,454 additions and 1 deletion.
2 changes: 2 additions & 0 deletions .github/ISSUE_TEMPLATE/bug-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ body:
required: false
- label: dbt-redshift
required: false
- label: dbt-snowflake
required: false
- label: dbt-spark
required: false
- type: textarea
Expand Down
2 changes: 2 additions & 0 deletions .github/ISSUE_TEMPLATE/regression-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ body:
required: false
- label: dbt-redshift
required: false
- label: dbt-snowflake
required: false
- label: dbt-spark
required: false
- type: textarea
Expand Down
2 changes: 2 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ updates:
- "/dbt-bigquery"
- "/dbt-postgres"
- "/dbt-redshift"
- "/dbt-snowflake"
- "/dbt-spark"
schedule:
interval: "daily"
Expand All @@ -22,6 +23,7 @@ updates:
- "/dbt-bigquery/docker"
- "/dbt-postgres/docker"
- "/dbt-redshift/docker"
- "/dbt-snowflake/docker"
- "/dbt-spark/docker"
schedule:
interval: "weekly"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_generate-changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
merge:
description: "Choose whether to merge the changelog branch"
Expand Down
33 changes: 33 additions & 0 deletions .github/workflows/_integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
branch:
description: "Choose the branch to test"
Expand Down Expand Up @@ -267,6 +268,38 @@ jobs:
- run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace
working-directory: ./${{ inputs.package }}

integration-tests-snowflake:
if: ${{ inputs.package == 'dbt-snowflake' }}
runs-on: ${{ inputs.os }}
environment:
name: ${{ inputs.package }}
env:
SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
SNOWFLAKE_TEST_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }}
SNOWFLAKE_TEST_DATABASE: ${{ vars.SNOWFLAKE_TEST_DATABASE }}
SNOWFLAKE_TEST_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WAREHOUSE }}
SNOWFLAKE_TEST_ALT_DATABASE: ${{ vars.SNOWFLAKE_TEST_ALT_DATABASE }}
SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ vars.SNOWFLAKE_TEST_QUOTED_DATABASE }}
SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ vars.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY }}
SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
repository: ${{ inputs.repository }}
- uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
- uses: pypa/hatch@install
- run: hatch run integration-tests tests/functional --ddtrace
working-directory: ./${{ inputs.package }}

integration-tests-spark:
if: ${{ inputs.package == 'dbt-spark' }}
runs-on: ${{ inputs.os }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_publish-internal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_publish-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
branch:
description: "Choose the branch to test"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_verify-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
branch:
description: "Choose the branch to build"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ on:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/pull-request-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ jobs:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
os: [ubuntu-22.04]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand All @@ -59,6 +60,7 @@ jobs:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
os: [ ubuntu-22.04 ]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand All @@ -80,6 +82,7 @@ jobs:
- "dbt-bigquery"
- "dbt-postgres"
- "dbt-redshift"
- "dbt-snowflake"
- "dbt-spark"
os: [ubuntu-22.04]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ repos:
rev: 7.1.1
hooks:
- id: flake8
exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests
exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests|dbt-snowflake/tests
args:
- --max-line-length=99
- --select=E,F,W
Expand Down
9 changes: 9 additions & 0 deletions dbt-snowflake/.changes/0.0.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
## Previous Releases
For information on prior major and minor releases, see their changelogs:
- [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md)
- [1.5](https://github.com/dbt-labs/dbt-snowflake/blob/1.5.latest/CHANGELOG.md)
- [1.4](https://github.com/dbt-labs/dbt-snowflake/blob/1.4.latest/CHANGELOG.md)
- [1.3](https://github.com/dbt-labs/dbt-snowflake/blob/1.3.latest/CHANGELOG.md)
- [1.2](https://github.com/dbt-labs/dbt-snowflake/blob/1.2.latest/CHANGELOG.md)
- [1.1](https://github.com/dbt-labs/dbt-snowflake/blob/1.1.latest/CHANGELOG.md)
- [1.0](https://github.com/dbt-labs/dbt-snowflake/blob/1.0.latest/CHANGELOG.md)
4 changes: 4 additions & 0 deletions dbt-snowflake/.changes/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# CHANGELOG


To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-snowflake/blob/main/.changes/README.md) found in `dbt-snowflake`.
6 changes: 6 additions & 0 deletions dbt-snowflake/.changes/header.tpl.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# dbt-snowflake Changelog

- This file provides a full account of all changes to `dbt-snowflake`.
- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry)
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Support MicrobatchConcurrency
time: 2024-12-02T09:51:36.606097-05:00
custom:
Author: michelleark
Issue: "1260"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Added support for custom iceberg base_location_root
time: 2025-01-13T13:34:14.326047-08:00
custom:
Author: LProcopi15
Issue: "1284"
6 changes: 6 additions & 0 deletions dbt-snowflake/.changes/unreleased/Fixes-20241018-173123.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Fix parsing of the VECTOR type
time: 2024-10-18T17:31:23.931299-04:00
custom:
Author: achawkins
Issue: "1098"
6 changes: 6 additions & 0 deletions dbt-snowflake/.changes/unreleased/Fixes-20241127-162204.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Use timestamp_tz type in microbatch `delete` DDL
time: 2024-11-27T16:22:04.103212-05:00
custom:
Author: michelleark
Issue: "1256"
6 changes: 6 additions & 0 deletions dbt-snowflake/.changes/unreleased/Fixes-20241209-131530.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: AUTO should no longer lead to rebuilds of dynamic tables.
time: 2024-12-09T13:15:30.554566-08:00
custom:
Author: versusfacit
Issue: "1267"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Move from setup.py to pyproject.toml and to hatch as a dev tool
time: 2024-11-17T18:44:30.85288-05:00
custom:
Author: mikealfare
Issue: "1250"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Use new `batch` context variables over `node.config.__dbt_internal` ones
time: 2024-12-05T14:40:36.373637-05:00
custom:
Author: michelleark
Issue: "1263"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Update default Python version for python models
time: 2024-12-11T17:08:31.842063-05:00
custom:
Author: mikealfare
Issue: "1203"
132 changes: 132 additions & 0 deletions dbt-snowflake/.changie.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
changesDir: .changes
unreleasedDir: unreleased
headerPath: header.tpl.md
versionHeaderPath: ""
changelogPath: CHANGELOG.md
versionExt: md
envPrefix: "CHANGIE_"
versionFormat: '## dbt-snowflake {{.Version}} - {{.Time.Format "January 02, 2006"}}'
kindFormat: '### {{.Kind}}'
changeFormat: |-
{{- $IssueList := list }}
{{- $changes := splitList " " $.Custom.Issue }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/issues/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
kinds:
- label: Breaking Changes
- label: Features
- label: Fixes
- label: Under the Hood
- label: Dependencies
changeFormat: |-
{{- $PRList := list }}
{{- $changes := splitList " " $.Custom.PR }}
{{- range $pullrequest := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $pullrequest }}
{{- $PRList = append $PRList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
skipGlobalChoices: true
additionalChoices:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: PR
label: GitHub Pull Request Number (separated by a single space if multiple)
type: string
minLength: 1
- label: Security
changeFormat: |-
{{- $PRList := list }}
{{- $changes := splitList " " $.Custom.PR }}
{{- range $pullrequest := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $pullrequest }}
{{- $PRList = append $PRList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
skipGlobalChoices: true
additionalChoices:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: PR
label: GitHub Pull Request Number (separated by a single space if multiple)
type: string
minLength: 1

newlines:
afterChangelogHeader: 1
afterKind: 1
afterChangelogVersion: 1
beforeKind: 1
endOfVersion: 1

custom:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: Issue
label: GitHub Issue Number (separated by a single space if multiple)
type: string
minLength: 1


footerFormat: |
{{- $contributorDict := dict }}
{{- /* ensure all names in this list are all lowercase for later matching purposes */}}
{{- $core_team := splitList " " .Env.CORE_TEAM }}
{{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
{{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
{{- range $team_member := $core_team }}
{{- $team_member_lower := lower $team_member }}
{{- $maintainers = append $maintainers $team_member_lower }}
{{- end }}
{{- range $change := .Changes }}
{{- $authorList := splitList " " $change.Custom.Author }}
{{- /* loop through all authors for a single changelog */}}
{{- range $author := $authorList }}
{{- $authorLower := lower $author }}
{{- /* we only want to include non-core team contributors */}}
{{- if not (has $authorLower $maintainers)}}
{{- $changeList := splitList " " $change.Custom.Author }}
{{- $IssueList := list }}
{{- $changeLink := $change.Kind }}
{{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
{{- $changes := splitList " " $change.Custom.PR }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/pull/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
{{- else }}
{{- $changes := splitList " " $change.Custom.Issue }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-snowflake/issues/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
{{- end }}
{{- /* check if this contributor has other changes associated with them already */}}
{{- if hasKey $contributorDict $author }}
{{- $contributionList := get $contributorDict $author }}
{{- $contributionList = concat $contributionList $IssueList }}
{{- $contributorDict := set $contributorDict $author $contributionList }}
{{- else }}
{{- $contributionList := $IssueList }}
{{- $contributorDict := set $contributorDict $author $contributionList }}
{{- end }}
{{- end}}
{{- end}}
{{- end }}
{{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}}
{{- if $contributorDict}}
### Contributors
{{- range $k,$v := $contributorDict }}
- [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}})
{{- end }}
{{- end }}
16 changes: 16 additions & 0 deletions dbt-snowflake/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# dbt-snowflake Changelog

- This file provides a full account of all changes to `dbt-snowflake`.
- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry)

## Previous Releases
For information on prior major and minor releases, see their changelogs:
- [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md)
- [1.5](https://github.com/dbt-labs/dbt-snowflake/blob/1.5.latest/CHANGELOG.md)
- [1.4](https://github.com/dbt-labs/dbt-snowflake/blob/1.4.latest/CHANGELOG.md)
- [1.3](https://github.com/dbt-labs/dbt-snowflake/blob/1.3.latest/CHANGELOG.md)
- [1.2](https://github.com/dbt-labs/dbt-snowflake/blob/1.2.latest/CHANGELOG.md)
- [1.1](https://github.com/dbt-labs/dbt-snowflake/blob/1.1.latest/CHANGELOG.md)
- [1.0](https://github.com/dbt-labs/dbt-snowflake/blob/1.0.latest/CHANGELOG.md)
Loading

0 comments on commit aa29250

Please sign in to comment.