dedup mitxonline staging models due to deleted records from source (#…

…1490)
mitodl · Feb 27, 2025 · 41a21d5 · 41a21d5
1 parent fa2d864
commit 41a21d5
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 3 deletions.
diff --git a/src/ol_dbt/models/staging/mitxonline/_stg_mitxonline__models.yml b/src/ol_dbt/models/staging/mitxonline/_stg_mitxonline__models.yml
@@ -69,7 +69,7 @@ models:
       column_list: ["courserun_id", "user_id"]
   - dbt_utils.recency:
       datepart: day
-      field: from_iso8601_timestamp(courserunenrollment_created_on)
+      field: from_iso8601_timestamp_nanos(courserunenrollment_created_on)
       interval: 2
       config:
         severity: error

diff --git a/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql b/src/ol_dbt/models/staging/mitxonline/stg__mitxonline__app__postgres__ecommerce_basket.sql
@@ -4,6 +4,8 @@ with source as (
 
 )
 
+{{ deduplicate_raw_table(order_by='id' , partition_columns = 'user_id') }}
+
 , renamed as (
 
     select
@@ -12,7 +14,7 @@ with source as (
         ,{{ cast_timestamp_to_iso8601('created_on') }} as basket_created_on
         ,{{ cast_timestamp_to_iso8601('updated_on') }} as basket_updated_on
 
-    from source
+    from most_recent_source
 
 )
 

diff --git a/...aging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql b/...aging/mitxonline/stg__mitxonline__app__postgres__flexiblepricing_currencyexchangerate.sql
@@ -5,6 +5,8 @@ with source as (
 
 )
 
+{{ deduplicate_raw_table(order_by='id' , partition_columns = 'currency_code') }}
+
 , renamed as (
 
     select
@@ -15,7 +17,7 @@ with source as (
         ,{{ cast_timestamp_to_iso8601('created_on') }} as currencyexchangerate_created_on
         ,{{ cast_timestamp_to_iso8601('updated_on') }} as currencyexchangerate_updated_on
 
-    from source
+    from most_recent_source
 
 )