07_gcp_dbt.yaml

id: 07_gcp_dbt
namespace: zoomcamp
inputs:
  - id: dbt_command
    type: SELECT
    allowCustomValue: true
    defaults: dbt build
    values:
      - dbt build
      - dbt debug # use when running the first time to validate DB connection

tasks:
  - id: sync
    type: io.kestra.plugin.git.SyncNamespaceFiles
    url: https://github.com/DataTalksClub/data-engineering-zoomcamp
    branch: main
    namespace: "{{flow.namespace}}"
    gitDirectory: 04-analytics-engineering/taxi_rides_ny
    dryRun: false
    # disabled: true # this Git Sync is needed only when running it the first time, afterwards the task can be disabled

  - id: dbt-build
    type: io.kestra.plugin.dbt.cli.DbtCLI
    env:
      DBT_DATABASE: "{{kv('GCP_PROJECT_ID')}}"
      DBT_SCHEMA: "{{kv('GCP_DATASET')}}"
    namespaceFiles:
      enabled: true
    containerImage: ghcr.io/kestra-io/dbt-bigquery:latest
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    inputFiles:
      sa.json: "{{kv('GCP_CREDS')}}"
    commands:
      - dbt deps
      - "{{ inputs.dbt_command }}"
    storeManifest:
      key: manifest.json
      namespace: "{{ flow.namespace }}"
    profiles: |
      default:
        outputs:
          dev:
            type: bigquery
            dataset: "{{kv('GCP_DATASET')}}"
            project: "{{kv('GCP_PROJECT_ID')}}"
            location: "{{kv('GCP_LOCATION')}}"
            keyfile: sa.json
            method: service-account
            priority: interactive
            threads: 16
            timeout_seconds: 300
            fixed_retries: 1
        target: dev
description: |
  Note that you need to adjust the models/staging/schema.yml file to match your database and schema. Select and edit that Namespace File from the UI. Save and run this flow. Once https://github.com/DataTalksClub/data-engineering-zoomcamp/pull/565/files is merged, you can ignore this note as it will be dynamically adjusted based on env variables.
  ```yaml
  sources:
    - name: staging
      database: kestra-sandbox 
      schema: zoomcamp
  ```