diff --git a/.github/workflows/rc031_models_run_preproduction.yml b/.github/workflows/rc031_models_run_preproduction.yml new file mode 100644 index 00000000..7e07d521 --- /dev/null +++ b/.github/workflows/rc031_models_run_preproduction.yml @@ -0,0 +1,47 @@ +name: rc031_models_run_preproduction + +on: + workflow_dispatch: + +env: + DBNAME: ${{ secrets.RC031_DBNAME_PROD }} + HOST_PREPROD: ${{ secrets.HOST_PREPROD }} + PASSWORD_PREPROD: ${{ secrets.PASSWORD_PREPROD }} + PORT_PREPROD: ${{ secrets.PORT_PREPROD }} + TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }} + USER_PREPROD: ${{ secrets.USER_PREPROD }} + PARTICIPATION_HOST_NAME: ${{ secrets.RC031_PARTICIPATION_HOST_NAME }} + WORKING_DIRECTORY: ./projects/rc031 + + +jobs: + rc031_models_run_preproduction: + name: rc031_models_run_preproduction + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@master + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.x" + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run dbt deps + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt deps --target preprod + + - name: Run dbt models + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt run --target preprod + + - name: Test dbt models + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt test --target preprod + diff --git a/.github/workflows/rc031_models_run_production.yml b/.github/workflows/rc031_models_run_production.yml new file mode 100644 index 00000000..7d3be5a8 --- /dev/null +++ b/.github/workflows/rc031_models_run_production.yml @@ -0,0 +1,48 @@ +name: rc031_models_run_production + +on: + workflow_dispatch: + +env: + DBNAME: ${{ secrets.RC031_DBNAME_PROD }} + HOST: ${{ secrets.HOST }} + PASSWORD: ${{ secrets.PASSWORD }} + PORT: ${{ secrets.PORT }} + TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }} + USER: ${{ secrets.USER }} + PARTICIPATION_HOST_NAME: ${{ secrets.RC031_PARTICIPATION_HOST_NAME }} + WORKING_DIRECTORY: ./projects/rc031 + + +jobs: + rc031_models_run: + name: rc031_models_run_production + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@master + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.x" + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run dbt deps + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt deps --target prod + + - name: Run dbt models + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt run --target prod + + - name: Test dbt models + working-directory: ${{ env.WORKING_DIRECTORY }} + run: dbt test --target prod + + diff --git a/projects/demo/.env.example b/projects/demo/.env.example index 118c1e8b..5ab49d05 100644 --- a/projects/demo/.env.example +++ b/projects/demo/.env.example @@ -1,6 +1,7 @@ DBNAME= -HOST= -PASSWORD= -PORT= +HOST_PREPROD= +PORT_PREPROD= +USER_PREPROD= +PASSWORD_PREPROD= TARGET_SCHEMA= -USER= \ No newline at end of file +PARTICIPATION_HOST_NAME= \ No newline at end of file diff --git a/projects/demo/macros/coalesce_legacy_and_new_columns.sql b/projects/demo/macros/coalesce_legacy_and_new_columns.sql index abc222ca..6a60d904 100644 --- a/projects/demo/macros/coalesce_legacy_and_new_columns.sql +++ b/projects/demo/macros/coalesce_legacy_and_new_columns.sql @@ -1,4 +1,4 @@ -{% macro coalesce_legacy_and_new_columns(relation, legacy_column_name, new_column_name) %} +{% macro coalesce_legacy_and_new_columns(relation, legacy_column_name, new_column_name, cast_type="TEXT") %} {% set columns = adapter.get_columns_in_relation(relation) %} {% set column_names = columns | map(attribute="name") %} @@ -7,7 +7,7 @@ {% elif new_column_name in column_names %} {{ new_column_name }} AS {{ legacy_column_name }} {% else %} - {{ get_column_if_exists(relation, legacy_column_name, cast_type="TEXT", default_value="NULL", include_alias=True) }} + {{ get_column_if_exists(relation, legacy_column_name, cast_type, default_value="NULL", include_alias=True) }} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/projects/demo/models/staging/decidim/forms/schema.yml b/projects/demo/models/staging/decidim/forms/schema.yml index 734b451f..58157e9b 100644 --- a/projects/demo/models/staging/decidim/forms/schema.yml +++ b/projects/demo/models/staging/decidim/forms/schema.yml @@ -5,6 +5,9 @@ sources: database: "{{ env_var('DBNAME') }}" schema: public tables: + - name: decidim_forms_response_options + - name: decidim_forms_response_choices + - name: decidim_forms_responses - name: decidim_forms_answer_options - name: decidim_forms_answer_choices - name: decidim_forms_answers diff --git a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_choices.sql b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_choices.sql index c86af2f5..64327c07 100644 --- a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_choices.sql +++ b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_choices.sql @@ -1,14 +1,14 @@ {% set relation = adapter.get_relation( database=target.database, schema='public', - identifier='decidim_form_response_choices' + identifier='decidim_forms_response_choices' ) %} {% if relation is not none %} SELECT id, decidim_response_id AS decidim_answer_id, - decidim_response_option_id AS decidim_answer_id, + decidim_response_option_id AS decidim_answer_option_id, position, REPLACE(SUBSTR(body, 2, length(body) - 2)::text, '\"', '"') AS body, custom_body, diff --git a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_options.sql b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_options.sql index fbee7116..3ffefc4a 100644 --- a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_options.sql +++ b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answer_options.sql @@ -3,7 +3,7 @@ {% set relation = adapter.get_relation( database=target.database, schema='public', - identifier='decidim_form_response_options' + identifier='decidim_forms_response_options' ) %} diff --git a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answers.sql b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answers.sql index 1077df17..989b120b 100644 --- a/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answers.sql +++ b/projects/demo/models/staging/decidim/forms/stg_decidim_forms_answers.sql @@ -1,7 +1,7 @@ {% set relation = adapter.get_relation( database=target.database, schema='public', - identifier='decidim_form_responses' + identifier='decidim_forms_responses' ) %} {% if relation is not none %} diff --git a/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql b/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql index a51404f6..cb361dd3 100644 --- a/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql +++ b/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql @@ -15,7 +15,7 @@ SELECT {{ stg_proposals_get_state(source('decidim', 'decidim_proposals_proposals')) }} AS state, {{ get_column_if_exists(source('decidim', 'decidim_proposals_proposals'), 'decidim_proposals_proposal_state_id', 'INTEGER') }}, comments_count, - {{ coalesce_legacy_and_new_columns(source('decidim', 'decidim_proposals_proposals'), 'endorsements_count', 'likes_count') }}, + {{ coalesce_legacy_and_new_columns(source('decidim', 'decidim_proposals_proposals'), 'endorsements_count', 'likes_count', 'BIGINT') }}, follows_count, address FROM {{ source('decidim', 'decidim_proposals_proposals') }} \ No newline at end of file diff --git a/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filter_items.sql b/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filter_items.sql index a0e1e1dc..bbda43bd 100644 --- a/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filter_items.sql +++ b/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filter_items.sql @@ -1,7 +1,7 @@ {% set relation = adapter.get_relation( database=target.database, schema='public', - identifier='decidim_taxonomizations' + identifier='decidim_taxonomy_filter_items' ) %} {% if relation is not none %} @@ -11,7 +11,7 @@ taxonomy_item_id, created_at, updated_at - FROM {{ source('decidim', 'decidim_taxonomizations') }} + FROM {{ source('decidim', 'decidim_taxonomy_filter_items') }} {% else %} SELECT CAST(NULL AS INTEGER) AS id, diff --git a/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filters.sql b/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filters.sql index 69eecb6a..8eeef2ca 100644 --- a/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filters.sql +++ b/projects/demo/models/staging/decidim/taxonomies/stg_decidim_taxonomy_filters.sql @@ -1,7 +1,7 @@ {% set relation = adapter.get_relation( database=target.database, schema='public', - identifier='decidim_taxonomizations' + identifier='decidim_taxonomy_filters' ) %} {% if relation is not none %} @@ -14,8 +14,8 @@ components_count, name, internal_name, - participatory_space_manifests, - FROM {{ source('decidim', 'decidim_taxonomizations') }} + participatory_space_manifests + FROM {{ source('decidim', 'decidim_taxonomy_filters') }} {% else %} SELECT CAST(NULL AS INTEGER) AS id, diff --git a/projects/rc031/config.yml b/projects/rc031/config.yml new file mode 100644 index 00000000..8f912129 --- /dev/null +++ b/projects/rc031/config.yml @@ -0,0 +1,3 @@ +db_name_dev: RC031_DBNAME_DEV +db_name_prod: RC031_DBNAME_PROD +participation_host_name: RC031_PARTICIPATION_HOST_NAME \ No newline at end of file diff --git a/projects/rc031/dbt_project.yml b/projects/rc031/dbt_project.yml new file mode 100644 index 00000000..efb42c64 --- /dev/null +++ b/projects/rc031/dbt_project.yml @@ -0,0 +1,34 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'rc031' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'rc031' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + rc031: + +materialized: table \ No newline at end of file diff --git a/projects/rc031/packages.yml b/projects/rc031/packages.yml new file mode 100644 index 00000000..f46b0908 --- /dev/null +++ b/projects/rc031/packages.yml @@ -0,0 +1,4 @@ +packages: + - local: ../demo + - package: dbt-labs/dbt_utils + version: 1.2.0 \ No newline at end of file diff --git a/projects/rc031/profiles.yml b/projects/rc031/profiles.yml new file mode 100644 index 00000000..6fdd8c0c --- /dev/null +++ b/projects/rc031/profiles.yml @@ -0,0 +1,30 @@ +rc031: + target: preprod + outputs: + dev: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST') }}" + pass: "{{ env_var('PASSWORD') }}" + port: "{{ env_var('PORT') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + threads: 2 + type: postgres + user: "{{ env_var('USER') }}" + preprod: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST_PREPROD') }}" + pass: "{{ env_var('PASSWORD_PREPROD') }}" + port: "{{ env_var('PORT_PREPROD') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + user: "{{ env_var('USER_PREPROD') }}" + threads: 2 + type: postgres + prod: + dbname: "{{ env_var('DBNAME') }}" + host: "{{ env_var('HOST') }}" + pass: "{{ env_var('PASSWORD') }}" + port: "{{ env_var('PORT') | as_number }}" + schema: "{{ env_var('TARGET_SCHEMA') }}" + threads: 2 + type: postgres + user: "{{ env_var('USER') }}"