add fields to marts__combined_course_enrollment_detail and upstream c…

…hanges (#992)

* add fields to combined enrollment mart and upstream changes

* Additional fields

* move tax field to staging

* update description and rename payment to receipt

* add payment_type

* fix discount_amount for xpro

* update description

src/ol_dbt/models/intermediate/bootcamps/_int_bootcamps__models.yml

@@ -51,26 +51,26 @@ models:
     tests:
     - not_null
   - name: receipt_reference_number
-    description: str, transaction reference number from user's cybersource payment
+    description: str, req_reference_number from user's cybersource receipt
     tests:
     - not_null
   - name: receipt_transaction_status
-    description: str, transaction status from cybersource payment. Value could be
+    description: str, transaction status from cybersource receipt. Value could be
       ACCEPT, CANCEL, ERROR, REVIEW or DECLINE.
     tests:
     - not_null
   - name: receipt_transaction_id
-    description: str, transaction identifier from cybersource payment. Maybe blank
+    description: str, transaction identifier from cybersource receipt. Maybe blank
       for CANCEL or ERROR transactions.
   - name: receipt_payment_method
-    description: str, payment method from cybersource transaction. Value could be
-      'paypal' or 'card' if payment is not cancelled
+    description: str, payment method from cybersource receipt. Value could be 'paypal'
+      or 'card' if payment is not cancelled
   - name: receipt_authorization_code
-    description: str, authorization code from cybersource payment
+    description: str, authorization code from cybersource receipt
   - name: receipt_bill_to_address_state
-    description: str, address state from cybersource payment
+    description: str, address state from cybersource receipt
   - name: receipt_bill_to_address_country
-    description: str, address country from cybersource payment
+    description: str, address country from cybersource receipt
   - name: order_id
     description: int, primary key in ecommerce_order
   tests:
@@ -194,6 +194,8 @@ models:
   - name: courserun_readable_id
     description: str, unique string to identify a bootcamp course run (could be blank
       for older runs)
+  - name: courserun_start_on
+    description: timestamp, date and time when the course starts
   - name: user_username
     description: str, name chosen by user
     tests:
@@ -204,6 +206,8 @@ models:
     - not_null
   - name: user_full_name
     description: string, full name on user's profile
+  - name: user_address_country
+    description: str, country code for the user's address
   tests:
   - dbt_expectations.expect_table_row_count_to_equal_other_table:
       compare_model: ref('stg__bootcamps__app__postgres__courserunenrollment')

src/ol_dbt/models/intermediate/bootcamps/int__bootcamps__courserunenrollments.sql

@@ -22,9 +22,11 @@ with enrollments as (
         , enrollments.courserunenrollment_enrollment_status
         , runs.courserun_readable_id
         , runs.courserun_title
+        , runs.courserun_start_on
         , users.user_username
         , users.user_email
         , users.user_full_name
+        , users.user_address_country
     from enrollments
     inner join runs on enrollments.courserun_id = runs.courserun_id
     inner join users on enrollments.user_id = users.user_id

src/ol_dbt/models/intermediate/edxorg/_int_edxorg__models.yml

@@ -129,6 +129,8 @@ models:
       that use new format
     tests:
     - not_null
+  - name: courserun_start_on
+    description: timestamp, date and time when the course starts
   - name: user_username
     description: str, username of a learner on the edX platform
     tests:

src/ol_dbt/models/intermediate/edxorg/int__edxorg__mitx_courserun_enrollments.sql

@@ -40,10 +40,7 @@ with person_courses as (
 )
 
 , edxorg_runs as (
-    select
-        courserun_readable_id
-        , course_number
-        , courserun_title
+    select *
     from {{ ref('stg__edxorg__bigquery__mitx_courserun') }}
 )
 
@@ -101,6 +98,7 @@ with person_courses as (
         , edxorg_users.user_username
         , micromasters_users.user_mitxonline_username
         , edxorg_runs.courserun_title
+        , edxorg_runs.courserun_start_date as courserun_start_on
         , edxorg_users.user_country as user_address_country
         , coalesce(edxorg_users.user_full_name, micromasters_users.user_full_name) as user_full_name
         , case

src/ol_dbt/models/intermediate/micromasters/_int_micromasters__models.yml

@@ -223,8 +223,12 @@ models:
     description: str, user full name on edX.org or MITxOnline
   - name: user_country
     description: str, user country on edX.org or MITxOnline
+  - name: courseruncertificate_uuid
+    description: str, unique identifier for the certificate on MITx Online, MicroMasters
+      or edX.org
   - name: courseruncertificate_url
-    description: str, the full URL to this DEDP course certificate on MITx Online
+    description: str, the full URL to the course certificate on mitxonline.mit.edu,
+      micromasters.mit.edu (legacy) or edX.org
   - name: courseruncertificate_created_on
     description: timestamp, date and time when this course certificate was initially
       created
@@ -436,6 +440,8 @@ models:
   - name: receipt_authorization_code
     description: str, authorization code from most recent cybersource payment for
       the order
+  - name: receipt_reference_number
+    description: str, req_reference_number from cybersource payment
   - name: receipt_bill_to_address_state
     description: str, address state from most recent cybersource payment for the order
   - name: receipt_bill_to_address_country
@@ -445,6 +451,12 @@ models:
     description: str, discount amount in readable format. It can be fixed discount
       which is $<dollar amount> off, fixed-price which is $<dollar amount> or percent-discount
       which is <percentage> % off
+  - name: discount_amount
+    description: numeric, actual discount dollar amount. For percent-discount coupon,
+      this is calculated as line_price * percentage off
+  - name: coupon_type
+    description: str, type of the coupon which describes what circumstances the coupon
+      can be redeemed. Possible values are "standard" or "discounted-previous-course".
   - name: coupon_code
     description: str, coupon code for the redeemed coupon
   - name: redeemedcoupon_created_on

src/ol_dbt/models/intermediate/micromasters/int__micromasters__course_certificates.sql

@@ -34,6 +34,7 @@ with course_certificates_dedp_from_micromasters as (
         , user_full_name
         , user_country
         , user_email
+        , coursecertificate_hash as courseruncertificate_uuid
         , coursecertificate_url as courseruncertificate_url
         , coursecertificate_created_on as courseruncertificate_created_on
     from course_certificates_dedp_from_micromasters
@@ -54,6 +55,7 @@ with course_certificates_dedp_from_micromasters as (
         , user_full_name
         , user_country
         , user_email
+        , courseruncertificate_uuid
         , courseruncertificate_url
         , courseruncertificate_created_on
     from course_certificates_dedp_from_mitxonline
@@ -75,6 +77,7 @@ with course_certificates_dedp_from_micromasters as (
         , user_full_name
         , user_country
         , user_email
+        , courseruncertificate_uuid
         , courseruncertificate_url
         , courseruncertificate_created_on
         , case
@@ -113,6 +116,7 @@ with course_certificates_dedp_from_micromasters as (
         , user_full_name
         , user_country
         , user_email
+        , courseruncertificate_uuid
         , courseruncertificate_url
         , courseruncertificate_created_on
     from dedp_course_certificates
@@ -132,6 +136,7 @@ with course_certificates_dedp_from_micromasters as (
         , user_full_name
         , user_country
         , user_email
+        , courseruncertificate_download_uuid as courseruncertificate_uuid
         , courseruncertificate_download_url as courseruncertificate_url
         , courseruncertificate_created_on
     from course_certificates_non_dedp_program

src/ol_dbt/models/intermediate/micromasters/int__micromasters__orders.sql

@@ -32,14 +32,21 @@ select
     , lines.line_price
     , lines.courserun_readable_id
     , lines.courserun_edxorg_readable_id
+    , receipts.receipt_reference_number
     , receipts.receipt_transaction_id
     , receipts.receipt_payment_method
     , receipts.receipt_authorization_code
     , receipts.receipt_bill_to_address_state
     , receipts.receipt_bill_to_address_country
+    , coupons.coupon_type
     , coupons.coupon_code
     , coupons.coupon_discount_amount_text
     , redeemedcoupons.redeemedcoupon_created_on
+    , case
+        when coupons.coupon_amount_type = 'percent-discount'
+            then cast(lines.line_price * coupons.coupon_amount as decimal(38, 2))
+        else cast(coupons.coupon_amount as decimal(38, 2))
+    end as coupon_amount
 from orders
 inner join lines on orders.order_id = lines.order_id
 left join redeemedcoupons on orders.order_id = redeemedcoupons.order_id

src/ol_dbt/models/intermediate/mitx/_int_mitx__models.yml

@@ -206,6 +206,11 @@ models:
     tests:
     - accepted_values:
         values: ['deferred', 'transferred', 'refunded', 'unenrolled']
+  - name: courserunenrollment_is_edx_enrolled
+    description: boolean, indicating whether the user is enrolled on edX platform.
+      For edx.org course enrollment, it would always be true.
+    tests:
+    - not_null
   - name: user_id
     description: int, user ID on the corresponding platform - MITx Online or edX.org
     tests:
@@ -219,6 +224,8 @@ models:
     - not_null
   - name: courserun_title
     description: str, title of the course run, maybe blank for some edX.org runs
+  - name: courserun_start_on
+    description: timestamp, date and time when the course starts
   - name: user_email
     description: str, current user email on edX.org or MITxOnline. For edx.org, learners
       can update their emails, so this is not necessary the same email as when learner
@@ -362,6 +369,9 @@ models:
       earned certificate for the course
     tests:
     - not_null
+  - name: courseruncertificate_uuid
+    description: str, unique identifier for the certificate on MITx Online, MicroMasters
+      or edX.org
   - name: courseruncertificate_url
     description: str, URL to the course certificate
     tests:

src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_certificates.sql

@@ -51,6 +51,7 @@ with mitxonline_certificates as (
         , course_number
         , courserun_title
         , courserun_readable_id
+        , courseruncertificate_uuid
         , courseruncertificate_url
         , courseruncertificate_created_on
         , user_username as user_mitxonline_username
@@ -66,6 +67,7 @@ with mitxonline_certificates as (
         , course_number
         , courserun_title
         , courserun_readable_id
+        , courseruncertificate_download_uuid as courseruncertificate_uuid
         , courseruncertificate_download_url as courseruncertificate_url
         , courseruncertificate_created_on
         , user_mitxonline_username
@@ -81,6 +83,7 @@ with mitxonline_certificates as (
         , course_number
         , courserun_title
         , courserun_readable_id
+        , courseruncertificate_uuid
         , courseruncertificate_url
         , courseruncertificate_created_on
         , user_mitxonline_username

src/ol_dbt/models/intermediate/mitx/int__mitx__courserun_enrollments.sql

@@ -23,10 +23,12 @@ with mitxonline_enrollments as (
         , courserunenrollment_created_on
         , courserunenrollment_enrollment_mode
         , courserunenrollment_enrollment_status
+        , courserunenrollment_is_edx_enrolled
         , courserun_id
         , courserun_title
         , courserun_readable_id
         , course_number
+        , courserun_start_on
         , user_id
         , user_email
         , user_full_name
@@ -45,10 +47,12 @@ with mitxonline_enrollments as (
         , courserunenrollment_created_on
         , courserunenrollment_enrollment_mode
         , null as courserunenrollment_enrollment_status
+        , true as courserunenrollment_is_edx_enrolled
         , null as courserun_id
         , courserun_title
         , courserun_readable_id
         , course_number
+        , courserun_start_on
         , user_id
         , user_email
         , user_full_name

src/ol_dbt/models/intermediate/mitxonline/_int_mitxonline__models.yml

@@ -436,6 +436,8 @@ models:
       May be blank for no payment required or refund.
   - name: transaction_authorization_code
     description: str, authorization code from cybersource payment
+  - name: receipt_reference_number
+    description: str, req_reference_number from cybersource payment
   - name: transaction_bill_to_address_state
     description: str, address state from cybersource payment
   - name: transaction_bill_to_address_country
@@ -515,10 +517,22 @@ models:
     description: int, primary key representing a single MITx Online course run
   - name: programrun_id
     description: int, primary key representing a single MITx Online program run
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+  - name: program_readable_id
+    description: str, Open edX ID formatted as program-v1:{org}+{program code}
+  - name: discount_source
+    description: str, type of discount coupon. Possible options are "marketing", "sales",
+      "financial-assistance", "customer-support", "staff", "legacy"
   - name: discount_amount_text
     description: str, discount amount in readable format from the most recent coupon
       redemption for the order. It can be percent-off which is <dollar amount>% off,
       dollars-off which is $<dollar amount> off or fixed-price
+  - name: discount_amount
+    description: numeric, actual discount dollar amount. For percent off discount,
+      this is calculated as product price * percentage off
+  - name: discount_redemption_type
+    description: str, one of  "one-time",  "one-time-per-user" and "unlimited"
   - name: discount_code
     description: str, discount code from the most recent coupon redemption for the
       order
@@ -532,6 +546,8 @@ models:
   - name: payment_method
     description: str, payment method from most recent cybersource payment. Value could
       be 'paypal' or 'card'.
+  - name: payment_req_reference_number
+    description: str, req_reference_number from cybersource payment
   - name: payment_bill_to_address_state
     description: str, address state from most recent cybersource payment
   - name: payment_bill_to_address_country
@@ -564,6 +580,10 @@ models:
     description: int, primary key representing a single MITx Online course run
   - name: programrun_id
     description: int, primary key representing a single MITx Online program run
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+  - name: program_readable_id
+    description: str, Open edX ID formatted as program-v1:{org}+{program code}
   - name: product_created_on
     description: timestamp, the timestamp the product was created
   - name: product_type
@@ -647,6 +667,8 @@ models:
     description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
     tests:
     - not_null
+  - name: courserun_start_on
+    description: timestamp, date and time when the course starts
   - name: user_username
     description: str, name chosen by user
     tests:

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__courserunenrollments.sql

@@ -109,6 +109,7 @@ with enrollments as (
         , mitxonline_runs.courserun_readable_id
         , mitxonline_runs.course_number
         , mitxonline_runs.course_id
+        , mitxonline_runs.courserun_start_on
         , mitxonline_users.user_username
         , mitxonline_users.user_email
         , mitxonline_users.user_edxorg_username

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_order.sql

@@ -73,14 +73,24 @@ select
     , intermediate_products_view.product_id
     , intermediate_products_view.courserun_id
     , intermediate_products_view.programrun_id
+    , intermediate_products_view.courserun_readable_id
+    , intermediate_products_view.program_readable_id
+    , discounts.discount_source
+    , discounts.discount_redemption_type
     , discounts.discount_code
     , discounts.discount_amount_text
     , discountredemptions.discountredemption_timestamp
     , payments.transaction_authorization_code as payment_authorization_code
     , payments.transaction_payment_method as payment_method
     , payments.transaction_readable_identifier as payment_transaction_id
+    , payments.transaction_reference_number as payment_req_reference_number
     , payments.transaction_bill_to_address_state as payment_bill_to_address_state
     , payments.transaction_bill_to_address_country as payment_bill_to_address_country
+    , case
+        when discounts.discount_type = 'percent-off'
+            then cast(intermediate_products_view.product_price * (discounts.discount_amount / 100) as decimal(38, 2))
+        else cast(discounts.discount_amount as decimal(38, 2))
+    end as discount_amount
 from lines
 inner join orders on lines.order_id = orders.order_id
 inner join users on orders.order_purchaser_user_id = users.user_id

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_product.sql

@@ -18,6 +18,10 @@ with products as (
     from {{ ref('stg__mitxonline__app__postgres__courses_programrun') }}
 )
 
+, programs as (
+    select * from {{ ref('stg__mitxonline__app__postgres__courses_program') }}
+)
+
 , product_subquery as (
     select
         products.product_id
@@ -43,6 +47,9 @@ select
     product_subquery.*
     , courseruns.course_id
     , programruns.program_id
+    , courseruns.courserun_readable_id
+    , programs.program_readable_id
 from product_subquery
 left join courseruns on product_subquery.courserun_id = courseruns.courserun_id
 left join programruns on product_subquery.programrun_id = programruns.programrun_id
+left join programs on programruns.program_id = programs.program_id

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__ecommerce_transaction.sql

@@ -14,6 +14,7 @@ select
     , transaction_status
     , transaction_authorization_code
     , transaction_payment_method
+    , transaction_reference_number
     , transaction_bill_to_address_state
     , transaction_bill_to_address_country
 from transactions