Skip to content

Commit 2e00a5b

Browse files
erikamoverikamov
committed
Test with old code
1 parent d2b97bd commit 2e00a5b

File tree

2 files changed

+51
-118
lines changed

2 files changed

+51
-118
lines changed

‎.github/workflows/deploy-dbt.yml‎

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -25,30 +25,9 @@ env:
2525
METABASE_DESTINATION_DATABASE: ${{ github.ref == 'refs/heads/main' && 'Data Marts (formerly Warehouse Views)' || '(Internal) Staging Warehouse Views' }}
2626

2727
jobs:
28-
models-changed:
29-
name: Detect dbt model changes
30-
runs-on: ubuntu-latest
31-
32-
outputs:
33-
any_changed: ${{ steps.changed-files-warehouse.outputs.any_changed }}
34-
35-
steps:
36-
- name: Checkout
37-
uses: actions/checkout@v4
38-
with:
39-
fetch-depth: 0
40-
41-
- uses: tj-actions/changed-files@v46
42-
id: changed-files-warehouse
43-
with:
44-
files: 'warehouse/models/**/*.sql'
45-
4628
compile:
4729
name: Compile dbt
4830
runs-on: ubuntu-latest
49-
needs: [models-changed]
50-
51-
if: ${{ needs.models-changed.outputs.any_changed == 'true' }}
5231

5332
permissions:
5433
contents: read
@@ -128,12 +107,30 @@ jobs:
128107
warehouse/target/*.json
129108
warehouse/target/*.html
130109
110+
models-changed:
111+
name: Detect dbt model changes
112+
runs-on: ubuntu-latest
113+
114+
outputs:
115+
any_changed: ${{ steps.changed-files-warehouse.outputs.any_changed }}
116+
117+
steps:
118+
- name: Checkout
119+
uses: actions/checkout@v4
120+
with:
121+
fetch-depth: 0
122+
123+
- uses: tj-actions/changed-files@v46
124+
id: changed-files-warehouse
125+
with:
126+
files: 'warehouse/models/**/*.sql'
127+
131128
metabase:
132129
name: Sync Metabase
133130
runs-on: ubuntu-latest
134-
needs: [models-changed, compile]
131+
needs: [compile, models-changed]
135132

136-
if: ${{ needs.models-changed.outputs.any_changed == 'true' }}
133+
# if: ${{ needs.models-changed.outputs.any_changed == 'true' }}
137134

138135
permissions:
139136
contents: read
@@ -208,10 +205,10 @@ jobs:
208205
if: ${{ env.DBT_TARGET == 'staging' }}
209206
run: gsutil cp -r gs://${{ env.DBT_ARTIFACTS_BUCKET }}/latest/ ./target/
210207

211-
- name: Run changed models
212-
working-directory: warehouse
213-
if: ${{ env.DBT_TARGET == 'staging' }}
214-
run: poetry run dbt run --select state:modified+ --target ${{ env.DBT_TARGET }} --state ./target/latest
208+
# - name: Run changed models
209+
# working-directory: warehouse
210+
# if: ${{ env.DBT_TARGET == 'staging' }}
211+
# run: poetry run dbt run --select state:modified+ --target ${{ env.DBT_TARGET }} --state ./target/latest
215212

216213
- name: Synchronize Metabase
217214
working-directory: warehouse
@@ -220,7 +217,7 @@ jobs:
220217
upload:
221218
name: Upload to Google Cloud Storage
222219
runs-on: ubuntu-latest
223-
needs: [models-changed, compile]
220+
needs: [compile, models-changed]
224221

225222
if: ${{ needs.models-changed.outputs.any_changed == 'true' }}
226223

@@ -308,7 +305,7 @@ jobs:
308305
visualize:
309306
name: Pull Request visualization
310307
runs-on: ubuntu-latest
311-
needs: [models-changed, compile]
308+
needs: [compile, models-changed]
312309

313310
if: ${{ github.event_name == 'pull_request' && needs.models-changed.outputs.any_changed == 'true' }}
314311

‎.github/workflows/extract-metabase-exposures.yml‎

Lines changed: 25 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ env:
1515
PROJECT_ID: ${{ github.ref == 'refs/heads/main' && 'cal-itp-data-infra' || 'cal-itp-data-infra-staging' }}
1616
DBT_TARGET: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }}
1717
DBT_ARTIFACTS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-artifacts' || 'calitp-staging-dbt-artifacts' }}
18+
DBT_DOCS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-docs' || 'calitp-staging-dbt-docs' }}
1819

1920
jobs:
20-
extract:
21-
name: Extract and Update Exposures
21+
compile:
22+
name: Compile and Extract Exposures
2223
runs-on: ubuntu-latest
2324

2425
permissions:
@@ -79,106 +80,41 @@ jobs:
7980
working-directory: warehouse
8081
run: poetry run dbt deps
8182

82-
- name: Get current timestamp
83-
uses: josStorer/get-current-time@v2
84-
id: current-time
85-
with:
86-
format: "YYYY-MM-DDTHH:mm:ss.SSSSSS+00:00"
87-
timezone: "UTC"
88-
89-
- name: Get current date
90-
uses: josStorer/get-current-time@v2
91-
id: current-date
92-
with:
93-
format: "YYYY-MM-DD"
94-
timezone: "UTC"
95-
9683
- name: Print dbt environment
9784
working-directory: warehouse
9885
run: poetry run dbt debug --target ${{ env.DBT_TARGET }}
9986

100-
- name: Compile dbt
87+
- name: Download latest artifacts from GCS
10188
working-directory: warehouse
102-
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
103-
104-
# - name: Download latest artifacts from GCS
105-
# working-directory: warehouse
106-
# run: gsutil cp -r gs://${{ env.DBT_ARTIFACTS_BUCKET }}/latest/manifest.json ./target
89+
run: gsutil cp -r gs://${{ env.DBT_ARTIFACTS_BUCKET }}/latest/manifest.json ./target
10790

108-
# - name: Create Metabase folder
109-
# working-directory: warehouse/models
110-
# run: mkdir metabase
91+
- name: Create Metabase folder
92+
working-directory: warehouse/models
93+
run: mkdir metabase
11194

112-
# - name: Extract Metabase exposures
113-
# working-directory: warehouse
114-
# run: poetry run dbt-metabase exposures -v --output-grouping="type" --output-path=./models/metabase/ --manifest-path=./target/manifest.json --metabase-url="https://dashboards.calitp.org" --metabase-api-key="${{ secrets.METABASE_API_KEY }}"
95+
- name: Extract Metabase exposures
96+
working-directory: warehouse
97+
run: poetry run dbt-metabase exposures -v --output-grouping="type" --output-path=./models/metabase/ --manifest-path=./target/manifest.json --metabase-url="https://dashboards.calitp.org" --metabase-api-key="${{ secrets.METABASE_API_KEY }}"
11598

116-
# - name: Compile dbt with exposures
117-
# working-directory: warehouse
118-
# run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
99+
- name: Compile dbt with exposures
100+
working-directory: warehouse
101+
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
119102

120103
- name: Generate dbt documentation
121104
working-directory: warehouse
122105
run: poetry run dbt docs generate --target ${{ env.DBT_TARGET }} --no-compile
123106

124-
- name: Upload Metabase exposures
125-
uses: google-github-actions/upload-cloud-storage@v1
107+
- name: Archive compilation artifacts
108+
uses: actions/upload-artifact@v4
126109
with:
127-
path: './warehouse/models/exposures/'
128-
parent: false
129-
process_gcloudignore: false
130-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/exposures/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
110+
name: dbt
111+
path: |
112+
warehouse/target/*.json
113+
warehouse/target/*.html
131114
132-
- name: Upload catalog.json
133-
uses: google-github-actions/upload-cloud-storage@v1
134-
with:
135-
path: './warehouse/target/'
136-
glob: 'catalog.json'
137-
parent: false
138-
process_gcloudignore: false
139-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/catalog.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
140-
141-
- name: Upload manifest.json
142-
uses: google-github-actions/upload-cloud-storage@v1
143-
with:
144-
path: './warehouse/target/'
145-
glob: 'manifest.json'
146-
parent: false
147-
process_gcloudignore: false
148-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/manifest.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
149-
150-
- name: Upload run_results.json
151-
uses: google-github-actions/upload-cloud-storage@v1
152-
with:
153-
path: './warehouse/target/'
154-
glob: 'run_results.json'
155-
parent: false
156-
process_gcloudignore: false
157-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/run_results.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
158-
159-
- name: Upload index.html
160-
uses: google-github-actions/upload-cloud-storage@v1
161-
with:
162-
path: './warehouse/target/'
163-
glob: 'index.html'
164-
parent: false
165-
process_gcloudignore: false
166-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/index.html/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
167-
168-
- name: Upload to latest
169-
uses: google-github-actions/upload-cloud-storage@v1
170-
with:
171-
path: './warehouse/target/'
172-
glob: '{catalog.json,manifest.json,index.html,run_results.json}'
173-
parent: false
174-
process_gcloudignore: false
175-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/"
176-
177-
- name: Upload documentation
178-
uses: google-github-actions/upload-cloud-storage@v1
115+
- name: Archive Metabase exposures
116+
uses: actions/upload-artifact@v4
179117
with:
180-
path: './warehouse/target/'
181-
glob: '{catalog.json,manifest.json,index.html}'
182-
parent: false
183-
process_gcloudignore: false
184-
destination: ${{ env.DBT_DOCS_BUCKET }}
118+
name: exposures
119+
path: |
120+
warehouse/models/metabase/*

0 commit comments

Comments
 (0)