Multivalue metafield macro support (#99)

* Multivalue metafield macro support * schema * Multivalue metafield macro support * vars commenting * macro simplification * yml updates * yml updates * add return to adapter.dispatch * PR updates * PR updates * PR updates
fivetran · Feb 5, 2025 · 0f418b6 · 0f418b6
1 parent d9df704
commit 0f418b6
Show file tree

Hide file tree

Showing 15 changed files with 97 additions and 41 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,14 @@
+# dbt_shopify v0.16.1
+[PR #99](https://github.com/fivetran/dbt_shopify/pull/99) includes the following updates:
+
+## Bug Fixes
+- Updated the `get_metafields` macro to support multiple reference values, ensuring compatibility with both the Shopify GraphQL API (ex: 'PRODUCTVARIANT' from the `product_variant` source) and the deprecated REST API (previously 'variant' for `product_variant`). [See the Shopify API docs for more information](https://shopify.dev/docs/api/admin-graphql/2025-01/objects/metafield).
+  - Updated `reference_value` parameter to `reference_values` to grab lists of reference values rather than a single string value. Now all records from `stg_shopify__metafield` are properly added to the relevant metafield models.
+  - Added `id_column` parameter to explicitly specify what field in the staging model should be joined on to properly unpivot the metafields. 
+
+## Under the Hood
+- Updated the `shopify_metafield_data` seed to validate the functionality of the `get_metafields` macro, ensuring it correctly retrieves metafield data for all supported reference values.
+
 # dbt_shopify v0.16.0
 This release includes the following updates:
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.16.0'
+version: '0.16.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.16.0'
+version: '0.16.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -9,11 +9,12 @@ clean-targets:         # directories to be removed by `dbt clean`
   - "dbt_modules"
 
 vars:
-  ## Uncomment when regenerating docs
+  # Uncomment when regenerating docs
   # shopify_using_fulfillment_event: true
   # shopify_using_all_metafields: true
   # shopify__standardized_billing_model_enabled: true
-  shopify_schema: shopify_integration_tests_15
+
+  shopify_schema: shopify_integration_tests_16
 
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
@@ -181,6 +182,8 @@ seeds:
       +column_types:
         value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
         type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        owner_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         _fivetran_synced: timestamp
         created_at: timestamp
         updated_at: timestamp

diff --git a/integration_tests/seeds/shopify_metafield_data.csv b/integration_tests/seeds/shopify_metafield_data.csv
@@ -1,6 +1,25 @@
 id,_fivetran_synced,created_at,description,key,namespace,owner_id,owner_resource,updated_at,value,value_type,type
 5445055,2022-11-19 10:06:09.531000,2019-10-28 20:06:39.000000,,returnAuthorizations,blade_runner,390244,order,2019-10-28 20:06:39.000000,"[{""id"":""ce95-49e4-9daf-41f29bbbb799"",""totalValue"":44444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444,""validReturnItems"":[{""UPC"":""19073825552"",""Quantity"":""1"",""Reason"":""changed-mind"",""LineItem"":""40055558892132""}]},""createdAt"":""2019-10-28T20:06:39.569Z"",""modifiedAt"":""2019-10-28T20:06:39.569Z""}]",,json_string
 6337647,2022-11-21 01:57:33.851000,2020-06-17 11:35:28.000000,,returnAuthorizations,blade_runner,254671,order,2020-06-17 11:35:28.000000,"[{""id"":""557ece73-658b-cf694dcd3f7e"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444.77,""validReturnItems"":[{""UPC"":""19055550468"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""4935555579471""}]},""createdAt"":""2020-06-17T11:35:28.469Z"",""modifiedAt"":""2020-06-17T11:35:28.470Z""}]",,json_string
-576111,2022-11-21 03:19:59.064000,2020-06-10 18:35:44.000000,,returnAuthorizations,blade_runner,22527,order,2020-06-10 18:35:44.000000,"[{""id"":""e461c20a-9dc7-d38de1c9012a"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444,""validReturnItems"":[{""UPC"":""190735551121"",""Quantity"":""1"",""Reason"":""too-big"",""LineItem"":""4925555231""}]},""createdAt"":""2020-06-10T18:35:44.043Z"",""modifiedAt"":""2020-06-10T18:35:44.043Z""}]",,json_string
-55241839,2022-11-21 01:29:09.347000,2020-07-15 21:24:16.000000,,returnAuthorizations,blade_runner,2335775,order,2020-07-15 21:24:16.000000,"[{""id"":""0c79163e-f55b56f50aff"",""totalValue"":44478.000000000004,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444.78000000000003,""validReturnItems"":[{""UPC"":""190555325"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""5555599407""}]},""createdAt"":""2020-07-15T21:24:16.210Z"",""modifiedAt"":""2020-07-15T21:24:16.210Z""}]",,json_string
-4575,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,220655,order,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+576111,2022-11-21 03:19:59.064000,2020-06-10 18:35:44.000000,,returnAuthorizations,blade_runner,2674098602081,order,2020-06-10 18:35:44.000000,"[{""id"":""e461c20a-9dc7-d38de1c9012a"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444,""validReturnItems"":[{""UPC"":""190735551121"",""Quantity"":""1"",""Reason"":""too-big"",""LineItem"":""4925555231""}]},""createdAt"":""2020-06-10T18:35:44.043Z"",""modifiedAt"":""2020-06-10T18:35:44.043Z""}]",,json_string
+55241839,2022-11-21 01:29:09.347000,2020-07-15 21:24:16.000000,,returnAuthorizations,blade_runner,2669516488801,order,2020-07-15 21:24:16.000000,"[{""id"":""0c79163e-f55b56f50aff"",""totalValue"":44478.000000000004,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444.78000000000003,""validReturnItems"":[{""UPC"":""190555325"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""5555599407""}]},""createdAt"":""2020-07-15T21:24:16.210Z"",""modifiedAt"":""2020-07-15T21:24:16.210Z""}]",,json_string
+4575,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,2674098602081,order,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+323013,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,41928883470372,variant,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+4135254,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,41928967323684,productvariant,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+4131434,2022-11-21 03:08:20.669000,2020-06-24 17:23:12.000000,,isInSpace,star_wars,2669516488801,ORDER,2020-06-24 17:23:12.000000,"true",,boolean
+9310039,2022-11-21 03:09:20.669000,2020-06-24 17:23:12.000000,,whenInSpace,star_wars,41928967323684,variant,2020-06-24 17:23:12.000000,1977-05-25,,date
+40032,2022-11-21 03:10:20.669000,2020-06-24 17:23:12.000000,,whereAreThey,avengers,41928967323684,PRODUCTVARIANT,2020-06-24 17:23:12.000000,"assembling",,string
+21408,2022-11-21 03:10:20.669000,2020-06-24 13:29:22.000000,,howMany,avengers,2674098602081,order,2020-06-24 17:23:12.000000,6,,integer
+2142325,2022-11-21 04:10:20.669000,2020-06-24 13:30:22.000000,,numberSequels,star_wars,2674098602081,order,2020-06-24 17:23:12.000000,9,,integer
+30000,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Boba Fett",,string
+30001,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Boba",,string
+30002,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,lastName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Fett",,string
+3000520,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Darth Vader",,string
+3000521,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Darth",,string
+3000522,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,lastName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Vader",,string
+3000523,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isJedi,star_wars,3589760876641,CUSTOMER,2020-06-24 17:23:12.000000,true,,boolean
+3000524,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isPhoto,star_wars,25974778232868,image,2020-06-24 17:23:12.000000,true,,boolean
+4009,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,"Chewbacca",,string
+4010,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,"Chewbacca",,string
+4011,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isJedi,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,false,,boolean
+4012,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isPhoto,star_wars,25974777741348,productimage,2020-06-24 17:23:12.000000,true,,boolean
diff --git a/macros/get_metafields.sql b/macros/get_metafields.sql
@@ -1,6 +1,22 @@
-{% macro get_metafields(source_object, reference_value, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %}
+{% macro get_metafields(source_object, reference_values, id_column, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %}
+    {{ return(adapter.dispatch('get_metafields', 'shopify')(source_object, reference_values, id_column, lookup_object, key_field, key_value, reference_field)) }} 
+{%- endmacro %}
 
-{% set pivot_fields = dbt_utils.get_column_values(table=ref(lookup_object), column=key_field, where="lower(" ~ reference_field ~ ") = lower('" ~ reference_value ~ "')") %}
+{% macro default__get_metafields(source_object, reference_values, id_column, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %}
+
+{# Manually quote and join reference values #}
+{% set quoted_values = [] %}
+{% for value in reference_values %}
+    {% do quoted_values.append("'" ~ value | lower | trim ~ "'") %}
+{% endfor %}
+{% set reference_values_clause = quoted_values | join(", ") %}
+
+{# Get the pivot fields dynamically based on the reference values #}
+{% set pivot_fields = dbt_utils.get_column_values(
+    table=ref(lookup_object),
+    column=key_field,
+    where="lower(" ~ reference_field ~ ") in (" ~ reference_values_clause ~ ")"
+) %}
 
 {% set source_columns = adapter.get_columns_in_relation(ref(source_object)) %}
 {% set source_column_count = source_columns | length %}
@@ -37,8 +53,8 @@ final as (
         {% endfor %}
     from source_table
     left join lookup_object 
-        on lookup_object.{{ reference_field }}_id = source_table.{{ reference_value }}_id
-        and lookup_object.{{ reference_field }} = '{{ reference_value }}'
+        on lookup_object.{{ reference_field }}_id = source_table.{{ id_column }}
+        and lower(lookup_object.{{ reference_field }}) in ({{ reference_values_clause }})
     {{ dbt_utils.group_by(source_column_count) }}
 )
 

diff --git a/models/metafields/shopify__collection_metafields.sql b/models/metafields/shopify__collection_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__collection", 
-    reference_value = 'collection') 
-}}
+    reference_values = ['collection'],
+    id_column = "collection_id"
+) }}
diff --git a/models/metafields/shopify__customer_metafields.sql b/models/metafields/shopify__customer_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) )}}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__customer", 
-    reference_value = 'customer') 
-}}
+    reference_values = ['customer'],
+    id_column = "customer_id"
+) }}
diff --git a/models/metafields/shopify__order_metafields.sql b/models/metafields/shopify__order_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__order", 
-    reference_value = 'order') 
-}}
+    reference_values = ['order'],
+    id_column = "order_id"
+) }}
diff --git a/models/metafields/shopify__product_image_metafields.sql b/models/metafields/shopify__product_image_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) ) }}
 
-{{ get_metafields( 
-    source_object = "stg_shopify__product_image", 
-    reference_value = 'image') 
-}}
+{{ shopify.get_metafields(
+    source_object = "stg_shopify__product_image",
+    reference_values = ['image', 'productimage'],
+    id_column = "product_image_id"
+) }}
diff --git a/models/metafields/shopify__product_metafields.sql b/models/metafields/shopify__product_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__product", 
-    reference_value = 'product') 
-}}
+    reference_values = ['product'],
+    id_column = "product_id"
+) }}
diff --git a/models/metafields/shopify__product_variant_metafields.sql b/models/metafields/shopify__product_variant_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) ) }}
 
-{{ get_metafields( 
-    source_object = "stg_shopify__product_variant", 
-    reference_value = 'variant') 
-}}
+{{ shopify.get_metafields( 
+    source_object = "stg_shopify__product_variant",
+    reference_values = ['variant', 'productvariant'],
+    id_column = "variant_id"
+) }}
diff --git a/models/metafields/shopify__shop_metafields.sql b/models/metafields/shopify__shop_metafields.sql
@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__shop", 
-    reference_value = 'shop') 
-}}
+    reference_values = ['shop'],
+    id_column = "shop_id"
+) }}