Skip to content

Commit 3379637

Browse files
erikamoverikamov
committed
Deploy dbt docs with Metabase Exposures
1 parent 749fda7 commit 3379637

File tree

1 file changed

+182
-0
lines changed

1 file changed

+182
-0
lines changed

‎.github/workflows/deploy-dbt-docs.yml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
name: Deploy dbt docs
2+
3+
on:
4+
# schedule:
5+
# - cron: "0 8 * * 1" # 8am UTC every Monday
6+
pull_request:
7+
paths:
8+
- .github/workflows/deploy-dbt-docs.yml
9+
- 'warehouse/**'
10+
11+
env:
12+
PYTHON_VERSION: '3.11'
13+
POETRY_VERSION: '2.0.1'
14+
SERVICE_ACCOUNT: ${{ github.ref == 'refs/heads/main' && 'github-actions-service-account@cal-itp-data-infra.iam.gserviceaccount.com' || 'github-actions-service-account@cal-itp-data-infra-staging.iam.gserviceaccount.com' }}
15+
WORKLOAD_IDENTITY_PROVIDER: ${{ github.ref == 'refs/heads/main' && 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions/providers/data-infra' || 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' }}
16+
PROJECT_ID: ${{ github.ref == 'refs/heads/main' && 'cal-itp-data-infra' || 'cal-itp-data-infra-staging' }}
17+
DBT_TARGET: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }}
18+
DBT_ARTIFACTS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-artifacts' || 'calitp-staging-dbt-artifacts' }}
19+
DBT_DOCS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-docs' || 'calitp-staging-dbt-docs' }}
20+
21+
jobs:
22+
compile:
23+
name: Compile dbt with Metabase Exposures
24+
runs-on: ubuntu-latest
25+
26+
permissions:
27+
contents: read
28+
id-token: write
29+
30+
steps:
31+
- name: Checkout
32+
uses: actions/checkout@v4
33+
34+
- name: Authenticate Google Service Account
35+
uses: google-github-actions/auth@v2
36+
with:
37+
create_credentials_file: 'true'
38+
project_id: ${{ env.PROJECT_ID }}
39+
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
40+
service_account: ${{ env.SERVICE_ACCOUNT }}
41+
42+
- name: Setup GCloud utilities
43+
uses: google-github-actions/setup-gcloud@v2
44+
45+
- name: Setup Graphviz
46+
uses: ts-graphviz/setup-graphviz@v2
47+
48+
- name: Setup Python
49+
uses: actions/setup-python@v5
50+
with:
51+
python-version: ${{ env.PYTHON_VERSION }}
52+
53+
- name: Cache Poetry
54+
uses: actions/cache@v3
55+
with:
56+
path: ~/.cache/pypoetry
57+
key: poetry-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-poetry-${{ env.POETRY_VERSION }}
58+
59+
- name: Setup Poetry
60+
uses: abatilo/actions-poetry@v3
61+
with:
62+
poetry-version: ${{ env.POETRY_VERSION }}
63+
64+
- name: Cache Python packages
65+
uses: actions/cache@v3
66+
with:
67+
path: ~/.local
68+
key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('warehouse/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }}
69+
70+
- name: Install dependencies
71+
working-directory: warehouse
72+
run: poetry install
73+
74+
- name: Cache dbt packages
75+
uses: actions/cache@v3
76+
with:
77+
path: warehouse/dbt_packages
78+
key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('warehouse/poetry.lock') }}-dbt-packages-${{ hashFiles('warehouse/packages.yml') }}
79+
80+
- name: Install dbt dependencies
81+
working-directory: warehouse
82+
run: poetry run dbt deps
83+
84+
- name: Print dbt environment
85+
working-directory: warehouse
86+
run: poetry run dbt debug --target ${{ env.DBT_TARGET }}
87+
88+
- name: Compile dbt
89+
working-directory: warehouse
90+
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
91+
92+
- name: Create Metabase folder
93+
working-directory: warehouse/models
94+
run: mkdir metabase
95+
96+
- name: Extract Metabase exposures
97+
working-directory: warehouse
98+
run: poetry run dbt-metabase exposures -v --output-grouping="type" --output-path=./models/metabase/ --manifest-path=./target/manifest.json --metabase-url="https://dashboards.calitp.org" --metabase-api-key="${{ secrets.METABASE_API_KEY }}"
99+
100+
- name: Compile dbt with exposures
101+
working-directory: warehouse
102+
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
103+
104+
- name: Generate dbt documentation
105+
working-directory: warehouse
106+
run: poetry run dbt docs generate --target ${{ env.DBT_TARGET }} --no-compile
107+
108+
- name: Get current timestamp
109+
uses: josStorer/get-current-time@v2
110+
id: current-time
111+
with:
112+
format: "YYYY-MM-DDTHH:mm:ss.SSSSSS+00:00"
113+
timezone: "UTC"
114+
115+
- name: Get current date
116+
uses: josStorer/get-current-time@v2
117+
id: current-date
118+
with:
119+
format: "YYYY-MM-DD"
120+
timezone: "UTC"
121+
122+
- name: Upload Metabase exposures
123+
uses: google-github-actions/upload-cloud-storage@v1
124+
with:
125+
path: './warehouse/models/exposures/'
126+
parent: false
127+
process_gcloudignore: false
128+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/exposures/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
129+
130+
- name: Upload catalog.json
131+
uses: google-github-actions/upload-cloud-storage@v1
132+
with:
133+
path: './warehouse/target/'
134+
glob: 'catalog.json'
135+
parent: false
136+
process_gcloudignore: false
137+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/catalog.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
138+
139+
- name: Upload manifest.json
140+
uses: google-github-actions/upload-cloud-storage@v1
141+
with:
142+
path: './warehouse/target/'
143+
glob: 'manifest.json'
144+
parent: false
145+
process_gcloudignore: false
146+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/manifest.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
147+
148+
- name: Upload run_results.json
149+
uses: google-github-actions/upload-cloud-storage@v1
150+
with:
151+
path: './warehouse/target/'
152+
glob: 'run_results.json'
153+
parent: false
154+
process_gcloudignore: false
155+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/run_results.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
156+
157+
- name: Upload index.html
158+
uses: google-github-actions/upload-cloud-storage@v1
159+
with:
160+
path: './warehouse/target/'
161+
glob: 'index.html'
162+
parent: false
163+
process_gcloudignore: false
164+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/index.html/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
165+
166+
- name: Upload to latest
167+
uses: google-github-actions/upload-cloud-storage@v1
168+
with:
169+
path: './warehouse/target/'
170+
glob: '{catalog.json,manifest.json,index.html,run_results.json}'
171+
parent: false
172+
process_gcloudignore: false
173+
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/"
174+
175+
- name: Upload documentation
176+
uses: google-github-actions/upload-cloud-storage@v1
177+
with:
178+
path: './warehouse/target/'
179+
glob: '{catalog.json,manifest.json,index.html}'
180+
parent: false
181+
process_gcloudignore: false
182+
destination: ${{ env.DBT_DOCS_BUCKET }}

0 commit comments

Comments
 (0)