Update MITx Online and MIT staging models to be compatible with the n…

…ew Airbyte synced raw tables (#1491)

* fix mitxonline mysql sourced staging build errors

* fix mitx mysql sourced staging build errors

src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__certificates_generatedcertificate.sql

@@ -1,9 +1,9 @@
-with certificates_generatedcertificate as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__certificates_generatedcertificate') }}
 )
 
-{{ deduplicate_query('certificates_generatedcertificate', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified_date' , partition_columns = 'id') }}
 
 select
     id

src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentcoursegrade.sql

@@ -1,8 +1,8 @@
-with grades_persistentcoursegrade as (
+with source as (
     select * from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__grades_persistentcoursegrade') }}
 )
 
-{{ deduplicate_query('grades_persistentcoursegrade', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 select
     course_id

src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__grades_persistentsubsectiongrade.sql

@@ -1,9 +1,9 @@
-with grades_persistentsubsectiongrade as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__grades_persistentsubsectiongrade') }}
 )
 
-{{ deduplicate_query('grades_persistentsubsectiongrade', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 
 select

src/ol_dbt/models/external/irx/mitx/irx__mitx__openedx__mysql__workflow_assessmentworkflow.sql

@@ -1,9 +1,9 @@
-with workflow_assessmentworkflow as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitx__openedx__mysql__workflow_assessmentworkflow') }}
 )
 
-{{ deduplicate_query('workflow_assessmentworkflow', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 select
     course_id

src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__certificates_generatedcertificate.sql

@@ -1,9 +1,9 @@
-with certificates_generatedcertificate as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__certificates_generatedcertificate') }}
 )
 
-{{ deduplicate_query('certificates_generatedcertificate', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified_date' , partition_columns = 'id') }}
 
 select
     id

src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentcoursegrade.sql

@@ -1,8 +1,8 @@
-with grades_persistentcoursegrade as (
+with source as (
     select * from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__grades_persistentcoursegrade') }}
 )
 
-{{ deduplicate_query('grades_persistentcoursegrade', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 select
     course_id

src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__grades_persistentsubsectiongrade.sql

@@ -1,9 +1,9 @@
-with grades_persistentsubsectiongrade as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade') }}
 )
 
-{{ deduplicate_query('grades_persistentsubsectiongrade', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 select
     course_id

src/ol_dbt/models/external/irx/mitxonline/irx__mitxonline__openedx__mysql__workflow_assessmentworkflow.sql

@@ -1,9 +1,9 @@
-with workflow_assessmentworkflow as (
+with source as (
     select *
     from {{ source('ol_warehouse_raw_data','raw__mitxonline__openedx__mysql__workflow_assessmentworkflow') }}
 )
 
-{{ deduplicate_query('workflow_assessmentworkflow', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 select
     course_id

src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__courseware_studentmodule.sql

@@ -2,7 +2,7 @@ with source as (
     select * from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__courseware_studentmodule') }}
 )
 
-{{ deduplicate_query('source', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 , cleaned as (
 
@@ -15,8 +15,8 @@ with source as (
         , state as studentmodule_state_data
         , grade as studentmodule_problem_grade
         , max_grade as studentmodule_problem_max_grade
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as studentmodule_created_on
-        , to_iso8601(from_iso8601_timestamp_nanos(modified)) as studentmodule_updated_on
+        , to_iso8601(created) as studentmodule_created_on
+        , to_iso8601(modified) as studentmodule_updated_on
     from most_recent_source
 )
 

src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongrade.sql

@@ -3,8 +3,7 @@ with source as (
     from {{ source('ol_warehouse_raw_data', 'raw__mitxonline__openedx__mysql__grades_persistentsubsectiongrade') }}
 )
 
-
-{{ deduplicate_query('source', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 , cleaned as (
 
@@ -18,9 +17,9 @@ with source as (
         , possible_graded as subsectiongrade_total_graded_score
         , earned_all as subsectiongrade_total_earned_score
         , earned_graded as subsectiongrade_total_earned_graded_score
-        , to_iso8601(from_iso8601_timestamp_nanos(first_attempted)) as subsectiongrade_first_attempted_on
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as subsectiongrade_created_on
-        , to_iso8601(from_iso8601_timestamp_nanos(modified)) as subsectiongrade_updated_on
+        , to_iso8601(first_attempted) as subsectiongrade_first_attempted_on
+        , to_iso8601(created) as subsectiongrade_created_on
+        , to_iso8601(modified) as subsectiongrade_updated_on
     from most_recent_source
 )
 

src/ol_dbt/models/staging/mitxonline/stg__mitxonline__openedx__mysql__grades_subsectiongradeoverride.sql

@@ -4,7 +4,7 @@ with source as (
    }}
 )
 
-{{ deduplicate_query('source', 'most_recent_source') }}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'id') }}
 
 , cleaned as (
 
@@ -17,8 +17,8 @@ with source as (
         , earned_graded_override as subsectiongradeoverride_total_earned_graded_score
         , override_reason as subsectiongradeoverride_reason
         , system as subsectiongradeoverride_system
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as subsectiongradeoverride_created_on
-        , to_iso8601(from_iso8601_timestamp_nanos(modified)) as subsectiongradeoverride_updated_on
+        , to_iso8601(created) as subsectiongradeoverride_created_on
+        , to_iso8601(modified) as subsectiongradeoverride_updated_on
     from most_recent_source
 )
 

src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__auth_user.sql

@@ -20,8 +20,8 @@ with source as (
                 ), '><', ''
             ), '<>', ' '
         ) as user_full_name
-        , to_iso8601(from_iso8601_timestamp_nanos(date_joined)) as user_joined_on
-        , to_iso8601(from_iso8601_timestamp_nanos(last_login)) as user_last_login
+        , to_iso8601(date_joined) as user_joined_on
+        , to_iso8601(last_login) as user_last_login
     from source
 )
 

src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courserun_enrollment.sql

@@ -9,7 +9,7 @@ with source as (
         , user_id
         , is_active as courserunenrollment_is_active
         , mode as courserunenrollment_enrollment_mode
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as courserunenrollment_created_on
+        , to_iso8601(created) as courserunenrollment_created_on
     from source
 )
 

src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__courseware_studentmodule.sql

@@ -2,12 +2,8 @@ with source as (
     select * from {{ source('ol_warehouse_raw_data', 'raw__mitx__openedx__mysql__courseware_studentmodule') }}
 )
 
-{{ deduplicate_query(
-   cte_name1='source',
-   cte_name2='most_recent_source' ,
-   partition_columns = 'course_id, student_id, module_id'
-   )
-}}
+{{ deduplicate_raw_table(order_by='modified' , partition_columns = 'course_id, student_id, module_id') }}
+
 
 , cleaned as (
 
@@ -20,8 +16,8 @@ with source as (
         , state as studentmodule_state_data
         , grade as studentmodule_problem_grade
         , max_grade as studentmodule_problem_max_grade
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as studentmodule_created_on
-        , to_iso8601(from_iso8601_timestamp_nanos(modified)) as studentmodule_updated_on
+        , to_iso8601(created) as studentmodule_created_on
+        , to_iso8601(modified) as studentmodule_updated_on
     from most_recent_source
 )
 

src/ol_dbt/models/staging/mitxresidential/stg__mitxresidential__openedx__edxval_video.sql

@@ -10,7 +10,7 @@ with source as (
         , client_video_id as video_client_id
         , status as video_status
         , duration as video_duration
-        , to_iso8601(from_iso8601_timestamp_nanos(created)) as video_created_on
+        , to_iso8601(created) as video_created_on
     from source
 )