03_postgres_dbt.yaml

id: 03_postgres_dbt
namespace: zoomcamp
inputs:
  - id: dbt_command
    type: SELECT
    allowCustomValue: true
    defaults: dbt build
    values:
      - dbt build
      - dbt debug # use when running the first time to validate DB connection
tasks:
  - id: sync
    type: io.kestra.plugin.git.SyncNamespaceFiles
    url: https://github.com/DataTalksClub/data-engineering-zoomcamp
    branch: main
    namespace: "{{ flow.namespace }}"
    gitDirectory: 04-analytics-engineering/taxi_rides_ny
    dryRun: false
    # disabled: true # this Git Sync is needed only when running it the first time, afterwards the task can be disabled

  - id: dbt-build
    type: io.kestra.plugin.dbt.cli.DbtCLI
    env:
      DBT_DATABASE: postgres-zoomcamp
      DBT_SCHEMA: public
    namespaceFiles:
      enabled: true
    containerImage: ghcr.io/kestra-io/dbt-postgres:latest
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    commands:
      - dbt deps
      - "{{ inputs.dbt_command }}"
    storeManifest:
      key: manifest.json
      namespace: "{{ flow.namespace }}"
    profiles: |
      default:
        outputs:
          dev:
            type: postgres
            host: host.docker.internal
            user: kestra
            password: k3str4
            port: 5432
            dbname: postgres-zoomcamp
            schema: public
            threads: 8
            connect_timeout: 10
            priority: interactive
        target: dev
description: |
  Note that you need to adjust the models/staging/schema.yml file to match your database and schema. Select and edit that Namespace File from the UI. Save and run this flow. Once https://github.com/DataTalksClub/data-engineering-zoomcamp/pull/565/files is merged, you can ignore this note as it will be dynamically adjusted based on env variables.
  ```yaml
  sources:
    - name: staging
      database: postgres-zoomcamp
      schema: public
  ```