diff --git a/src/ol_dbt/models/dimensional/_dim_course_run.yml b/src/ol_dbt/models/dimensional/_dim_course_run.yml index 57954246d..b96f159ff 100644 --- a/src/ol_dbt/models/dimensional/_dim_course_run.yml +++ b/src/ol_dbt/models/dimensional/_dim_course_run.yml @@ -77,6 +77,21 @@ models: description: > Timestamp when the course run was created in the source system. Populated for mitxonline, mitxpro, and residential platforms. Null for edxorg and bootcamps. + - name: semester + description: > + Academic term identifier for a MicroMasters proctored exam run (e.g. "2T2022"). + Populated only for MITxOnline course runs that correspond to a MicroMasters + exam + run. Null for all other platforms and non-exam MITxOnline course runs. + - name: passing_grade + description: > + Passing score threshold (0.0–1.0) set per MicroMasters proctored exam run. + Populated only for MITxOnline course runs that correspond to a MicroMasters + exam + run. Null for all other platforms and non-exam MITxOnline course runs. Note: + a + stored value of null and an actual passing grade of 0.0 are distinct; -1.0 is + used as the SCD2 change-detection sentinel to distinguish them. - name: effective_date description: > Timestamp (UTC) when this SCD Type 2 version became effective. Always set for diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index b75bd1f6b..a28f9b041 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -5,21 +5,45 @@ on_schema_change='append_new_columns' ) }} -with mitxonline_courseruns as ( +-- MicroMasters stores exam run metadata (semester label and passing grade threshold) +-- keyed by examrun_readable_id, which matches the MITxOnline courserun_readable_id for +-- proctored exam course runs. These are the only platform-specific dimensional attributes +-- that originate outside the platform's own course run record. +with micromasters_examruns as ( + select examrun_readable_id, examrun_semester, examrun_passing_grade + from ( + select + examrun_readable_id + , examrun_semester + , examrun_passing_grade + , row_number() over ( + partition by examrun_readable_id + order by examrun_updated_on desc nulls last, examrun_id desc + ) as _row_num + from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} + ) as deduped_examruns + where _row_num = 1 +) + +, mitxonline_courseruns as ( select - courserun_readable_id - , courserun_id as source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , courserun_is_live - , courserun_created_on + cr.courserun_readable_id + , cr.courserun_id as source_id + , cr.course_id + , cr.courserun_title + , cr.courserun_start_on + , cr.courserun_end_on + , cr.courserun_enrollment_start_on as enrollment_start + , cr.courserun_enrollment_end_on as enrollment_end + , cr.courserun_is_live + , cr.courserun_created_on , cast(null as varchar) as course_readable_id + , er.examrun_semester as semester + , er.examrun_passing_grade as passing_grade , 'mitxonline' as platform - from {{ ref('int__mitxonline__course_runs') }} + from {{ ref('int__mitxonline__course_runs') }} as cr + left join micromasters_examruns as er + on cr.courserun_readable_id = er.examrun_readable_id ) , mitxpro_courseruns as ( @@ -35,6 +59,8 @@ with mitxonline_courseruns as ( , courserun_is_live , courserun_created_on , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade , 'mitxpro' as platform from {{ ref('int__mitxpro__course_runs') }} ) @@ -52,6 +78,8 @@ with mitxonline_courseruns as ( , courserun_is_published as courserun_is_live , cast(null as varchar) as courserun_created_on , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade , 'edxorg' as platform from {{ ref('int__edxorg__mitx_courseruns') }} ) @@ -69,6 +97,8 @@ with mitxonline_courseruns as ( , cast(null as boolean) as courserun_is_live , courserun_created_on , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade , 'residential' as platform from {{ ref('int__mitxresidential__courseruns') }} ) @@ -86,6 +116,8 @@ with mitxonline_courseruns as ( , false as courserun_is_live , cast(null as varchar) as courserun_created_on , runs.course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade , 'bootcamps' as platform from {{ ref('int__bootcamps__course_runs') }} as runs ) @@ -116,6 +148,8 @@ with mitxonline_courseruns as ( , courserun_is_live , courserun_created_on , platform + , semester + , passing_grade , coalesce( course_readable_id, case @@ -198,6 +232,8 @@ with mitxonline_courseruns as ( , enrollment_end , courserun_is_live , courserun_created_on + , semester + , passing_grade , current_timestamp as effective_date , cast(null as timestamp) as end_date , true as is_current @@ -217,6 +253,8 @@ with mitxonline_courseruns as ( and coalesce(existing.enrollment_start, '') = coalesce(courseruns_with_all_fks.enrollment_start, '') and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') and coalesce(existing.courserun_is_live, false) = coalesce(courseruns_with_all_fks.courserun_is_live, false) + and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') + and coalesce(existing.passing_grade, -1.0) = coalesce(courseruns_with_all_fks.passing_grade, -1.0) ) {% endif %} ) @@ -242,6 +280,8 @@ with mitxonline_courseruns as ( , existing.enrollment_end , existing.courserun_is_live , existing.courserun_created_on + , existing.semester + , existing.passing_grade , existing.effective_date , current_timestamp as end_date , false as is_current diff --git a/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml b/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml index 8e8c476a4..69f9c589e 100644 --- a/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml +++ b/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml @@ -39,9 +39,8 @@ models: tests: - not_null - name: semester - description: str, semester for the exam, e.g., 2T2022. - tests: - - not_null + description: str, semester for the exam, e.g., 2T2022. Null for MITx Online exam + runs that do not have a corresponding MicroMasters exam run record. tests: - dbt_expectations.expect_compound_columns_to_be_unique: arguments: