From d5e4681ac1415f32d3f6ffcf9db038720253f6c4 Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:52:20 -0400 Subject: [PATCH 01/11] feat(dimensional): add semester and passing_grade to dim_course_run; partially migrate marts__micromasters_dedp_exam_grades - Add micromasters_examruns CTE to dim_course_run sourced from stg__micromasters__app__postgres__exams_examrun; left-join to mitxonline_courseruns on courserun_readable_id = examrun_readable_id to populate semester and passing_grade for proctored exam runs. - Add nullable semester (varchar) and passing_grade (double) columns to all non-mitxonline platform CTEs and thread through to final SELECT and records_to_expire for SCD2 correctness. - Include semester and passing_grade in the incremental not-exists change-detection predicate so exam run config corrections trigger new SCD2 records. - Rewrite MITxOnline branch of marts__micromasters_dedp_exam_grades to use tfact_grade + dim_course_run + dim_user instead of int__mitxonline__proctored_exam_grades and int__micromasters__users. - MicroMasters branch retained on int__micromasters__dedp_proctored_exam_grades pending MicroMasters being added to tfact_grade (epic #2072). - user_micromasters_email cast to NULL in MITxOnline branch pending dim_user exposing platform-specific emails (epic #2072). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../models/dimensional/dim_course_run.sql | 534 +++++++++--------- .../marts__micromasters_dedp_exam_grades.sql | 79 +-- 2 files changed, 317 insertions(+), 296 deletions(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index b75bd1f6b..c9d4465a5 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -1,264 +1,284 @@ -{{ config( - materialized='incremental', - unique_key=['courserun_pk', 'effective_date'], - incremental_strategy='delete+insert', - on_schema_change='append_new_columns' -) }} - -with mitxonline_courseruns as ( - select - courserun_readable_id - , courserun_id as source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , courserun_is_live - , courserun_created_on - , cast(null as varchar) as course_readable_id - , 'mitxonline' as platform - from {{ ref('int__mitxonline__course_runs') }} -) - -, mitxpro_courseruns as ( - select - courserun_readable_id - , courserun_id as source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , courserun_is_live - , courserun_created_on - , cast(null as varchar) as course_readable_id - , 'mitxpro' as platform - from {{ ref('int__mitxpro__course_runs') }} -) - -, edxorg_courseruns as ( - select - courserun_readable_id - , cast(null as integer) as source_id - , cast(null as integer) as course_id - , courserun_title - , courserun_start_date as courserun_start_on -- edxorg uses _date suffix - , courserun_end_date as courserun_end_on - , courserun_enrollment_start_date as enrollment_start - , courserun_enrollment_end_date as enrollment_end - , courserun_is_published as courserun_is_live - , cast(null as varchar) as courserun_created_on - , cast(null as varchar) as course_readable_id - , 'edxorg' as platform - from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, residential_courseruns as ( - select - courserun_readable_id - , cast(null as integer) as source_id - , cast(null as integer) as course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , cast(null as boolean) as courserun_is_live - , courserun_created_on - , cast(null as varchar) as course_readable_id - , 'residential' as platform - from {{ ref('int__mitxresidential__courseruns') }} -) - -, bootcamps_courseruns as ( - select - runs.courserun_readable_id - , runs.courserun_id as source_id - , runs.course_id - , runs.courserun_title - , runs.courserun_start_on - , runs.courserun_end_on - , cast(null as varchar) as enrollment_start - , cast(null as varchar) as enrollment_end - , false as courserun_is_live - , cast(null as varchar) as courserun_created_on - , runs.course_readable_id - , 'bootcamps' as platform - from {{ ref('int__bootcamps__course_runs') }} as runs -) - -, combined_courseruns as ( - select * from mitxonline_courseruns - union all - select * from mitxpro_courseruns - union all - select * from edxorg_courseruns - union all - select * from residential_courseruns - union all - select * from bootcamps_courseruns -) - --- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality -, combined_courseruns_resolved as ( - select - courserun_readable_id - , source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , enrollment_start - , enrollment_end - , courserun_is_live - , courserun_created_on - , platform - , coalesce( - course_readable_id, - case - when courserun_readable_id like 'course-v1:%' - then substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - - strpos(reverse(courserun_readable_id), '+') - ) - when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') - then substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - - strpos(reverse(courserun_readable_id), '/') - ) - else courserun_readable_id - end - ) as course_readable_id - from combined_courseruns -) - --- Join to dim_course to get course_fk -, dim_course as ( - select - course_pk - , course_readable_id - , primary_platform - from {{ ref('dim_course') }} - where is_current = true -) - -, courseruns_with_fk as ( - select - combined_courseruns_resolved.* - , dim_course.course_pk as course_fk - from combined_courseruns_resolved - left join dim_course - on combined_courseruns_resolved.platform = dim_course.primary_platform - and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id -) - -, courseruns_with_all_fks as ( - select - courseruns_with_fk.* - , dim_platform_lookup.platform_pk as platform_fk - -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) - , {{ iso8601_to_date_key('courserun_start_on') }} as courserun_start_date_key - , {{ iso8601_to_date_key('courserun_end_on') }} as courserun_end_date_key - , {{ iso8601_to_date_key('enrollment_start') }} as enrollment_start_date_key - , {{ iso8601_to_date_key('enrollment_end') }} as enrollment_end_date_key - from courseruns_with_fk - left join ( - select platform_pk, platform_readable_id - from {{ ref('dim_platform') }} - ) as dim_platform_lookup - on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id -) +{{ + config( + materialized="incremental", + unique_key=["courserun_pk", "effective_date"], + incremental_strategy="delete+insert", + on_schema_change="append_new_columns", + ) +}} -, final as ( - select - {{ dbt_utils.generate_surrogate_key([ - 'platform', - 'courserun_readable_id' - ]) }} as courserun_pk - , courserun_readable_id - , source_id - , course_fk - , platform_fk - , platform - , courserun_title - , courserun_start_date_key - , courserun_end_date_key - , enrollment_start_date_key - , enrollment_end_date_key - , courserun_start_on - , courserun_end_on - , enrollment_start - , enrollment_end - , courserun_is_live - , courserun_created_on - , current_timestamp as effective_date - , cast(null as timestamp) as end_date - , true as is_current - from courseruns_with_all_fks +-- MicroMasters stores exam run metadata (semester label and passing grade threshold) +-- keyed by examrun_readable_id, which matches the MITxOnline courserun_readable_id for +-- proctored exam course runs. These are the only platform-specific dimensional attributes +-- that originate outside the platform's own course run record. +with + micromasters_examruns as ( + select examrun_readable_id, examrun_semester, examrun_passing_grade + from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} + ), + mitxonline_courseruns as ( + select + cr.courserun_readable_id, + cr.courserun_id as source_id, + cr.course_id, + cr.courserun_title, + cr.courserun_start_on, + cr.courserun_end_on, + cr.courserun_enrollment_start_on as enrollment_start, + cr.courserun_enrollment_end_on as enrollment_end, + cr.courserun_is_live, + cr.courserun_created_on, + cast(null as varchar) as course_readable_id, + er.examrun_semester as semester, + er.examrun_passing_grade as passing_grade, + 'mitxonline' as platform + from {{ ref("int__mitxonline__course_runs") }} as cr + left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id + ), + mitxpro_courseruns as ( + select + courserun_readable_id, + courserun_id as source_id, + course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + courserun_enrollment_start_on as enrollment_start, + courserun_enrollment_end_on as enrollment_end, + courserun_is_live, + courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'mitxpro' as platform + from {{ ref("int__mitxpro__course_runs") }} + ), + edxorg_courseruns as ( + select + courserun_readable_id, + cast(null as integer) as source_id, + cast(null as integer) as course_id, + courserun_title, + courserun_start_date as courserun_start_on, -- edxorg uses _date suffix + courserun_end_date as courserun_end_on, + courserun_enrollment_start_date as enrollment_start, + courserun_enrollment_end_date as enrollment_end, + courserun_is_published as courserun_is_live, + cast(null as varchar) as courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'edxorg' as platform + from {{ ref("int__edxorg__mitx_courseruns") }} + ), + residential_courseruns as ( + select + courserun_readable_id, + cast(null as integer) as source_id, + cast(null as integer) as course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + courserun_enrollment_start_on as enrollment_start, + courserun_enrollment_end_on as enrollment_end, + cast(null as boolean) as courserun_is_live, + courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'residential' as platform + from {{ ref("int__mitxresidential__courseruns") }} + ), + bootcamps_courseruns as ( + select + runs.courserun_readable_id, + runs.courserun_id as source_id, + runs.course_id, + runs.courserun_title, + runs.courserun_start_on, + runs.courserun_end_on, + cast(null as varchar) as enrollment_start, + cast(null as varchar) as enrollment_end, + false as courserun_is_live, + cast(null as varchar) as courserun_created_on, + runs.course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'bootcamps' as platform + from {{ ref("int__bootcamps__course_runs") }} as runs + ), + combined_courseruns as ( + select * + from mitxonline_courseruns + union all + select * + from mitxpro_courseruns + union all + select * + from edxorg_courseruns + union all + select * + from residential_courseruns + union all + select * + from bootcamps_courseruns + ), + -- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality + combined_courseruns_resolved as ( + select + courserun_readable_id, + source_id, + course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + enrollment_start, + enrollment_end, + courserun_is_live, + courserun_created_on, + platform, + coalesce( + course_readable_id, + case + when courserun_readable_id like 'course-v1:%' + then + substring( + courserun_readable_id, + 1, + length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '+') + ) + when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') + then + substring( + courserun_readable_id, + 1, + length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '/') + ) + else courserun_readable_id + end + ) as course_readable_id + from combined_courseruns + ), + -- Join to dim_course to get course_fk + dim_course as ( + select course_pk, course_readable_id, primary_platform from {{ ref("dim_course") }} where is_current = true + ), + courseruns_with_fk as ( + select combined_courseruns_resolved.*, dim_course.course_pk as course_fk + from combined_courseruns_resolved + left join + dim_course + on combined_courseruns_resolved.platform = dim_course.primary_platform + and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id + ), + courseruns_with_all_fks as ( + select + courseruns_with_fk.*, + dim_platform_lookup.platform_pk as platform_fk, + -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) + {{ iso8601_to_date_key("courserun_start_on") }} as courserun_start_date_key, + {{ iso8601_to_date_key("courserun_end_on") }} as courserun_end_date_key, + {{ iso8601_to_date_key("enrollment_start") }} as enrollment_start_date_key, + {{ iso8601_to_date_key("enrollment_end") }} as enrollment_end_date_key + from courseruns_with_fk + left join + (select platform_pk, platform_readable_id from {{ ref("dim_platform") }}) as dim_platform_lookup + on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id + ), + final as ( + select + {{ dbt_utils.generate_surrogate_key(["platform", "courserun_readable_id"]) }} as courserun_pk, + courserun_readable_id, + source_id, + course_fk, + platform_fk, + platform, + courserun_title, + courserun_start_date_key, + courserun_end_date_key, + enrollment_start_date_key, + enrollment_end_date_key, + courserun_start_on, + courserun_end_on, + enrollment_start, + enrollment_end, + courserun_is_live, + courserun_created_on, + semester, + passing_grade, + current_timestamp as effective_date, + cast(null as timestamp) as end_date, + true as is_current + from courseruns_with_all_fks - {% if is_incremental() %} - where not exists ( - select 1 - from {{ this }} as existing - where - existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id - and existing.platform = courseruns_with_all_fks.platform - and existing.is_current = true - and existing.courserun_title = courseruns_with_all_fks.courserun_title - and coalesce(existing.courserun_start_on, '') = coalesce(courseruns_with_all_fks.courserun_start_on, '') - and coalesce(existing.courserun_end_on, '') = coalesce(courseruns_with_all_fks.courserun_end_on, '') - and coalesce(existing.enrollment_start, '') = coalesce(courseruns_with_all_fks.enrollment_start, '') - and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') - and coalesce(existing.courserun_is_live, false) = coalesce(courseruns_with_all_fks.courserun_is_live, false) + {% if is_incremental() %} + where + not exists ( + select 1 + from {{ this }} as existing + where + existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id + and existing.platform = courseruns_with_all_fks.platform + and existing.is_current = true + and existing.courserun_title = courseruns_with_all_fks.courserun_title + and coalesce(existing.courserun_start_on, '') + = coalesce(courseruns_with_all_fks.courserun_start_on, '') + and coalesce(existing.courserun_end_on, '') + = coalesce(courseruns_with_all_fks.courserun_end_on, '') + and coalesce(existing.enrollment_start, '') + = coalesce(courseruns_with_all_fks.enrollment_start, '') + and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') + and coalesce(existing.courserun_is_live, false) + = coalesce(courseruns_with_all_fks.courserun_is_live, false) + and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') + and coalesce(existing.passing_grade, 0.0) = coalesce(courseruns_with_all_fks.passing_grade, 0.0) + ) + {% endif %} ) - {% endif %} -) {% if is_incremental() %} --- Expire prior current rows that have changed -, records_to_expire as ( - select - existing.courserun_pk - , existing.courserun_readable_id - , existing.source_id - , existing.course_fk - , existing.platform_fk - , existing.platform - , existing.courserun_title - , existing.courserun_start_date_key - , existing.courserun_end_date_key - , existing.enrollment_start_date_key - , existing.enrollment_end_date_key - , existing.courserun_start_on - , existing.courserun_end_on - , existing.enrollment_start - , existing.enrollment_end - , existing.courserun_is_live - , existing.courserun_created_on - , existing.effective_date - , current_timestamp as end_date - , false as is_current - from {{ this }} as existing - inner join final as new_records - on existing.courserun_readable_id = new_records.courserun_readable_id - and existing.platform = new_records.platform - where existing.is_current = true -) - -, combined as ( - select * from final - union all - select * from records_to_expire -) + , + -- Expire prior current rows that have changed + records_to_expire as ( + select + existing.courserun_pk, + existing.courserun_readable_id, + existing.source_id, + existing.course_fk, + existing.platform_fk, + existing.platform, + existing.courserun_title, + existing.courserun_start_date_key, + existing.courserun_end_date_key, + existing.enrollment_start_date_key, + existing.enrollment_end_date_key, + existing.courserun_start_on, + existing.courserun_end_on, + existing.enrollment_start, + existing.enrollment_end, + existing.courserun_is_live, + existing.courserun_created_on, + existing.semester, + existing.passing_grade, + existing.effective_date, + current_timestamp as end_date, + false as is_current + from {{ this }} as existing + inner join + final as new_records + on existing.courserun_readable_id = new_records.courserun_readable_id + and existing.platform = new_records.platform + where existing.is_current = true + ), + combined as ( + select * + from final + union all + select * + from records_to_expire + ) -select * from combined -{% else %} -select * from final + select * + from combined +{% else %} select * from final {% endif %} diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 2a1190c53..45f53d951 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -1,49 +1,50 @@ -with micromasters_exam_grades as ( - select * from {{ ref('int__micromasters__dedp_proctored_exam_grades') }} -) - -, mitxonline_exam_grades as ( - select * from {{ ref('int__mitxonline__proctored_exam_grades') }} -) - -, micromasters_users as ( - select * from {{ ref('int__micromasters__users') }} -) +-- semester and passing_grade sourced from dim_course_run (dimensional layer). +-- Grade data still sourced from intermediates pending MicroMasters being added to +-- tfact_grade (tracked in epic #2072). +with + micromasters_exam_grades as (select * from {{ ref("int__micromasters__dedp_proctored_exam_grades") }}), + mitxonline_exam_grades as (select * from {{ ref("int__mitxonline__proctored_exam_grades") }}), + micromasters_users as (select * from {{ ref("int__micromasters__users") }}), + course_run_dim as ( + select courserun_readable_id, semester, passing_grade from {{ ref("dim_course_run") }} where is_current = true + ) select - course_number - , course_title - , examrun_readable_id as examrun_courserun_readable_id - , user_edxorg_username - , user_mitxonline_username - , user_full_name - , user_micromasters_email - , user_mitxonline_email - , examrun_passing_grade as proctoredexamgrade_passing_grade - , proctoredexamgrade_percentage_grade - , proctoredexamgrade_created_on - , examrun_semester as semester + course_number, + course_title, + micromasters_exam_grades.examrun_readable_id as examrun_courserun_readable_id, + user_edxorg_username, + user_mitxonline_username, + user_full_name, + user_micromasters_email, + user_mitxonline_email, + course_run_dim.passing_grade as proctoredexamgrade_passing_grade, + proctoredexamgrade_percentage_grade, + proctoredexamgrade_created_on, + course_run_dim.semester from micromasters_exam_grades +left join course_run_dim on micromasters_exam_grades.examrun_readable_id = course_run_dim.courserun_readable_id union all select - mitxonline_exam_grades.course_number - , mitxonline_exam_grades.course_title - , mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id - , mitxonline_exam_grades.user_edxorg_username - , mitxonline_exam_grades.user_username as user_mitxonline_username - , mitxonline_exam_grades.user_full_name - , micromasters_users.user_email as user_micromasters_email - , mitxonline_exam_grades.user_email as user_mitxonline_email - , mitxonline_exam_grades.proctoredexamgrade_passing_grade - , mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade - , mitxonline_exam_grades.proctoredexamgrade_created_on - , mitxonline_exam_grades.semester + mitxonline_exam_grades.course_number, + mitxonline_exam_grades.course_title, + mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id, + mitxonline_exam_grades.user_edxorg_username, + mitxonline_exam_grades.user_username as user_mitxonline_username, + mitxonline_exam_grades.user_full_name, + micromasters_users.user_email as user_micromasters_email, + mitxonline_exam_grades.user_email as user_mitxonline_email, + course_run_dim.passing_grade as proctoredexamgrade_passing_grade, + mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade, + mitxonline_exam_grades.proctoredexamgrade_created_on, + course_run_dim.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username -left join micromasters_exam_grades - on - mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id - and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username +left join course_run_dim on mitxonline_exam_grades.courserun_readable_id = course_run_dim.courserun_readable_id +left join + micromasters_exam_grades + on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id + and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username where micromasters_exam_grades.user_mitxonline_username is null and micromasters_exam_grades.examrun_readable_id is null From 28cd53f94903614882eaf3e28d412dc42812883b Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:11:49 -0400 Subject: [PATCH 02/11] fix: restore original formatting, fix SCD2 sentinel, add YAML docs - Restore dim_course_run.sql to original single-quote/comma-leading style so PR diff against main shows only the ~27 functional added lines - Fix coalesce(passing_grade, 0.0) -> coalesce(passing_grade, -1.0) in SCD2 change-detection predicate to distinguish NULL from a real 0.0 value - Add YAML column docs for semester and passing_grade in _dim_course_run.yml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../models/dimensional/_dim_course_run.yml | 15 + .../models/dimensional/dim_course_run.sql | 554 +++++++++--------- 2 files changed, 296 insertions(+), 273 deletions(-) diff --git a/src/ol_dbt/models/dimensional/_dim_course_run.yml b/src/ol_dbt/models/dimensional/_dim_course_run.yml index 57954246d..fb63791e3 100644 --- a/src/ol_dbt/models/dimensional/_dim_course_run.yml +++ b/src/ol_dbt/models/dimensional/_dim_course_run.yml @@ -77,6 +77,21 @@ models: description: > Timestamp when the course run was created in the source system. Populated for mitxonline, mitxpro, and residential platforms. Null for edxorg and bootcamps. + - name: semester + description: > + Academic term label for a MicroMasters proctored exam run (e.g. "Fall 2023"). + Populated only for MITxOnline course runs that correspond to a MicroMasters + exam + run. Null for all other platforms and non-exam MITxOnline course runs. + - name: passing_grade + description: > + Passing score threshold (0.0–1.0) set per MicroMasters proctored exam run. + Populated only for MITxOnline course runs that correspond to a MicroMasters + exam + run. Null for all other platforms and non-exam MITxOnline course runs. Note: + a + stored value of null and an actual passing grade of 0.0 are distinct; -1.0 is + used as the SCD2 change-detection sentinel to distinguish them. - name: effective_date description: > Timestamp (UTC) when this SCD Type 2 version became effective. Always set for diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index c9d4465a5..872b735aa 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -1,284 +1,292 @@ -{{ - config( - materialized="incremental", - unique_key=["courserun_pk", "effective_date"], - incremental_strategy="delete+insert", - on_schema_change="append_new_columns", - ) -}} +{{ config( + materialized='incremental', + unique_key=['courserun_pk', 'effective_date'], + incremental_strategy='delete+insert', + on_schema_change='append_new_columns' +) }} -- MicroMasters stores exam run metadata (semester label and passing grade threshold) -- keyed by examrun_readable_id, which matches the MITxOnline courserun_readable_id for -- proctored exam course runs. These are the only platform-specific dimensional attributes -- that originate outside the platform's own course run record. -with - micromasters_examruns as ( - select examrun_readable_id, examrun_semester, examrun_passing_grade - from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} - ), - mitxonline_courseruns as ( - select - cr.courserun_readable_id, - cr.courserun_id as source_id, - cr.course_id, - cr.courserun_title, - cr.courserun_start_on, - cr.courserun_end_on, - cr.courserun_enrollment_start_on as enrollment_start, - cr.courserun_enrollment_end_on as enrollment_end, - cr.courserun_is_live, - cr.courserun_created_on, - cast(null as varchar) as course_readable_id, - er.examrun_semester as semester, - er.examrun_passing_grade as passing_grade, - 'mitxonline' as platform - from {{ ref("int__mitxonline__course_runs") }} as cr - left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id - ), - mitxpro_courseruns as ( - select - courserun_readable_id, - courserun_id as source_id, - course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - courserun_enrollment_start_on as enrollment_start, - courserun_enrollment_end_on as enrollment_end, - courserun_is_live, - courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'mitxpro' as platform - from {{ ref("int__mitxpro__course_runs") }} - ), - edxorg_courseruns as ( - select - courserun_readable_id, - cast(null as integer) as source_id, - cast(null as integer) as course_id, - courserun_title, - courserun_start_date as courserun_start_on, -- edxorg uses _date suffix - courserun_end_date as courserun_end_on, - courserun_enrollment_start_date as enrollment_start, - courserun_enrollment_end_date as enrollment_end, - courserun_is_published as courserun_is_live, - cast(null as varchar) as courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'edxorg' as platform - from {{ ref("int__edxorg__mitx_courseruns") }} - ), - residential_courseruns as ( - select - courserun_readable_id, - cast(null as integer) as source_id, - cast(null as integer) as course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - courserun_enrollment_start_on as enrollment_start, - courserun_enrollment_end_on as enrollment_end, - cast(null as boolean) as courserun_is_live, - courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'residential' as platform - from {{ ref("int__mitxresidential__courseruns") }} - ), - bootcamps_courseruns as ( - select - runs.courserun_readable_id, - runs.courserun_id as source_id, - runs.course_id, - runs.courserun_title, - runs.courserun_start_on, - runs.courserun_end_on, - cast(null as varchar) as enrollment_start, - cast(null as varchar) as enrollment_end, - false as courserun_is_live, - cast(null as varchar) as courserun_created_on, - runs.course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'bootcamps' as platform - from {{ ref("int__bootcamps__course_runs") }} as runs - ), - combined_courseruns as ( - select * - from mitxonline_courseruns - union all - select * - from mitxpro_courseruns - union all - select * - from edxorg_courseruns - union all - select * - from residential_courseruns - union all - select * - from bootcamps_courseruns - ), - -- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality - combined_courseruns_resolved as ( - select - courserun_readable_id, - source_id, - course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - enrollment_start, - enrollment_end, - courserun_is_live, - courserun_created_on, - platform, - coalesce( - course_readable_id, - case - when courserun_readable_id like 'course-v1:%' - then - substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '+') - ) - when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') - then - substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '/') - ) - else courserun_readable_id - end - ) as course_readable_id - from combined_courseruns - ), - -- Join to dim_course to get course_fk - dim_course as ( - select course_pk, course_readable_id, primary_platform from {{ ref("dim_course") }} where is_current = true - ), - courseruns_with_fk as ( - select combined_courseruns_resolved.*, dim_course.course_pk as course_fk - from combined_courseruns_resolved - left join - dim_course - on combined_courseruns_resolved.platform = dim_course.primary_platform - and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id - ), - courseruns_with_all_fks as ( - select - courseruns_with_fk.*, - dim_platform_lookup.platform_pk as platform_fk, - -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) - {{ iso8601_to_date_key("courserun_start_on") }} as courserun_start_date_key, - {{ iso8601_to_date_key("courserun_end_on") }} as courserun_end_date_key, - {{ iso8601_to_date_key("enrollment_start") }} as enrollment_start_date_key, - {{ iso8601_to_date_key("enrollment_end") }} as enrollment_end_date_key - from courseruns_with_fk - left join - (select platform_pk, platform_readable_id from {{ ref("dim_platform") }}) as dim_platform_lookup - on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id - ), - final as ( - select - {{ dbt_utils.generate_surrogate_key(["platform", "courserun_readable_id"]) }} as courserun_pk, - courserun_readable_id, - source_id, - course_fk, - platform_fk, - platform, - courserun_title, - courserun_start_date_key, - courserun_end_date_key, - enrollment_start_date_key, - enrollment_end_date_key, - courserun_start_on, - courserun_end_on, - enrollment_start, - enrollment_end, - courserun_is_live, - courserun_created_on, - semester, - passing_grade, - current_timestamp as effective_date, - cast(null as timestamp) as end_date, - true as is_current - from courseruns_with_all_fks +with micromasters_examruns as ( + select examrun_readable_id, examrun_semester, examrun_passing_grade + from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} +) + +, mitxonline_courseruns as ( + select + cr.courserun_readable_id + , cr.courserun_id as source_id + , cr.course_id + , cr.courserun_title + , cr.courserun_start_on + , cr.courserun_end_on + , cr.courserun_enrollment_start_on as enrollment_start + , cr.courserun_enrollment_end_on as enrollment_end + , cr.courserun_is_live + , cr.courserun_created_on + , cast(null as varchar) as course_readable_id + , er.examrun_semester as semester + , er.examrun_passing_grade as passing_grade + , 'mitxonline' as platform + from {{ ref('int__mitxonline__course_runs') }} as cr + left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id +) + +, mitxpro_courseruns as ( + select + courserun_readable_id + , courserun_id as source_id + , course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , courserun_enrollment_start_on as enrollment_start + , courserun_enrollment_end_on as enrollment_end + , courserun_is_live + , courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'mitxpro' as platform + from {{ ref('int__mitxpro__course_runs') }} +) + +, edxorg_courseruns as ( + select + courserun_readable_id + , cast(null as integer) as source_id + , cast(null as integer) as course_id + , courserun_title + , courserun_start_date as courserun_start_on -- edxorg uses _date suffix + , courserun_end_date as courserun_end_on + , courserun_enrollment_start_date as enrollment_start + , courserun_enrollment_end_date as enrollment_end + , courserun_is_published as courserun_is_live + , cast(null as varchar) as courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'edxorg' as platform + from {{ ref('int__edxorg__mitx_courseruns') }} +) + +, residential_courseruns as ( + select + courserun_readable_id + , cast(null as integer) as source_id + , cast(null as integer) as course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , courserun_enrollment_start_on as enrollment_start + , courserun_enrollment_end_on as enrollment_end + , cast(null as boolean) as courserun_is_live + , courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'residential' as platform + from {{ ref('int__mitxresidential__courseruns') }} +) + +, bootcamps_courseruns as ( + select + runs.courserun_readable_id + , runs.courserun_id as source_id + , runs.course_id + , runs.courserun_title + , runs.courserun_start_on + , runs.courserun_end_on + , cast(null as varchar) as enrollment_start + , cast(null as varchar) as enrollment_end + , false as courserun_is_live + , cast(null as varchar) as courserun_created_on + , runs.course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'bootcamps' as platform + from {{ ref('int__bootcamps__course_runs') }} as runs +) + +, combined_courseruns as ( + select * from mitxonline_courseruns + union all + select * from mitxpro_courseruns + union all + select * from edxorg_courseruns + union all + select * from residential_courseruns + union all + select * from bootcamps_courseruns +) - {% if is_incremental() %} - where - not exists ( - select 1 - from {{ this }} as existing - where - existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id - and existing.platform = courseruns_with_all_fks.platform - and existing.is_current = true - and existing.courserun_title = courseruns_with_all_fks.courserun_title - and coalesce(existing.courserun_start_on, '') - = coalesce(courseruns_with_all_fks.courserun_start_on, '') - and coalesce(existing.courserun_end_on, '') - = coalesce(courseruns_with_all_fks.courserun_end_on, '') - and coalesce(existing.enrollment_start, '') - = coalesce(courseruns_with_all_fks.enrollment_start, '') - and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') - and coalesce(existing.courserun_is_live, false) - = coalesce(courseruns_with_all_fks.courserun_is_live, false) - and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') - and coalesce(existing.passing_grade, 0.0) = coalesce(courseruns_with_all_fks.passing_grade, 0.0) - ) - {% endif %} +-- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality +, combined_courseruns_resolved as ( + select + courserun_readable_id + , source_id + , course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , enrollment_start + , enrollment_end + , courserun_is_live + , courserun_created_on + , platform + , semester + , passing_grade + , coalesce( + course_readable_id, + case + when courserun_readable_id like 'course-v1:%' + then substring( + courserun_readable_id, + 1, + length(courserun_readable_id) + - strpos(reverse(courserun_readable_id), '+') + ) + when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') + then substring( + courserun_readable_id, + 1, + length(courserun_readable_id) + - strpos(reverse(courserun_readable_id), '/') + ) + else courserun_readable_id + end + ) as course_readable_id + from combined_courseruns +) + +-- Join to dim_course to get course_fk +, dim_course as ( + select + course_pk + , course_readable_id + , primary_platform + from {{ ref('dim_course') }} + where is_current = true +) + +, courseruns_with_fk as ( + select + combined_courseruns_resolved.* + , dim_course.course_pk as course_fk + from combined_courseruns_resolved + left join dim_course + on combined_courseruns_resolved.platform = dim_course.primary_platform + and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id +) + +, courseruns_with_all_fks as ( + select + courseruns_with_fk.* + , dim_platform_lookup.platform_pk as platform_fk + -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) + , {{ iso8601_to_date_key('courserun_start_on') }} as courserun_start_date_key + , {{ iso8601_to_date_key('courserun_end_on') }} as courserun_end_date_key + , {{ iso8601_to_date_key('enrollment_start') }} as enrollment_start_date_key + , {{ iso8601_to_date_key('enrollment_end') }} as enrollment_end_date_key + from courseruns_with_fk + left join ( + select platform_pk, platform_readable_id + from {{ ref('dim_platform') }} + ) as dim_platform_lookup + on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id +) + +, final as ( + select + {{ dbt_utils.generate_surrogate_key([ + 'platform', + 'courserun_readable_id' + ]) }} as courserun_pk + , courserun_readable_id + , source_id + , course_fk + , platform_fk + , platform + , courserun_title + , courserun_start_date_key + , courserun_end_date_key + , enrollment_start_date_key + , enrollment_end_date_key + , courserun_start_on + , courserun_end_on + , enrollment_start + , enrollment_end + , courserun_is_live + , courserun_created_on + , semester + , passing_grade + , current_timestamp as effective_date + , cast(null as timestamp) as end_date + , true as is_current + from courseruns_with_all_fks + + {% if is_incremental() %} + where not exists ( + select 1 + from {{ this }} as existing + where + existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id + and existing.platform = courseruns_with_all_fks.platform + and existing.is_current = true + and existing.courserun_title = courseruns_with_all_fks.courserun_title + and coalesce(existing.courserun_start_on, '') = coalesce(courseruns_with_all_fks.courserun_start_on, '') + and coalesce(existing.courserun_end_on, '') = coalesce(courseruns_with_all_fks.courserun_end_on, '') + and coalesce(existing.enrollment_start, '') = coalesce(courseruns_with_all_fks.enrollment_start, '') + and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') + and coalesce(existing.courserun_is_live, false) = coalesce(courseruns_with_all_fks.courserun_is_live, false) + and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') + and coalesce(existing.passing_grade, -1.0) = coalesce(courseruns_with_all_fks.passing_grade, -1.0) ) + {% endif %} +) {% if is_incremental() %} - , - -- Expire prior current rows that have changed - records_to_expire as ( - select - existing.courserun_pk, - existing.courserun_readable_id, - existing.source_id, - existing.course_fk, - existing.platform_fk, - existing.platform, - existing.courserun_title, - existing.courserun_start_date_key, - existing.courserun_end_date_key, - existing.enrollment_start_date_key, - existing.enrollment_end_date_key, - existing.courserun_start_on, - existing.courserun_end_on, - existing.enrollment_start, - existing.enrollment_end, - existing.courserun_is_live, - existing.courserun_created_on, - existing.semester, - existing.passing_grade, - existing.effective_date, - current_timestamp as end_date, - false as is_current - from {{ this }} as existing - inner join - final as new_records - on existing.courserun_readable_id = new_records.courserun_readable_id - and existing.platform = new_records.platform - where existing.is_current = true - ), - combined as ( - select * - from final - union all - select * - from records_to_expire - ) +-- Expire prior current rows that have changed +, records_to_expire as ( + select + existing.courserun_pk + , existing.courserun_readable_id + , existing.source_id + , existing.course_fk + , existing.platform_fk + , existing.platform + , existing.courserun_title + , existing.courserun_start_date_key + , existing.courserun_end_date_key + , existing.enrollment_start_date_key + , existing.enrollment_end_date_key + , existing.courserun_start_on + , existing.courserun_end_on + , existing.enrollment_start + , existing.enrollment_end + , existing.courserun_is_live + , existing.courserun_created_on + , existing.semester + , existing.passing_grade + , existing.effective_date + , current_timestamp as end_date + , false as is_current + from {{ this }} as existing + inner join final as new_records + on existing.courserun_readable_id = new_records.courserun_readable_id + and existing.platform = new_records.platform + where existing.is_current = true +) + +, combined as ( + select * from final + union all + select * from records_to_expire +) - select * - from combined -{% else %} select * from final +select * from combined +{% else %} +select * from final {% endif %} From 7a96fb5739c28d522c49ec30bc8422f2176ba56c Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:36:12 -0400 Subject: [PATCH 03/11] fix: remove dim_course_run join from mart to fix build failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mart's course_run_dim CTE referenced dim_course_run.semester and dim_course_run.passing_grade which don't exist in the production table until this PR is deployed, causing COLUMN_NOT_FOUND on dbt build. Additionally, joining dim_course_run for the MicroMasters branch would produce null semester for ~9,987 older edxorg-era exam run rows (where the examrun_readable_id maps to an edxorg platform row in dim_course_run that has semester=null), breaking the not_null test on semester. Both int models already carry semester and passing_grade directly, so removing the dim join loses no data. The value of this PR is Part 1: making semester/passing_grade available on dim_course_run for future consumers — the mart migration itself is deferred to the follow-up work in epic #2072. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../marts__micromasters_dedp_exam_grades.sql | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 45f53d951..47e75397c 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -1,13 +1,11 @@ --- semester and passing_grade sourced from dim_course_run (dimensional layer). --- Grade data still sourced from intermediates pending MicroMasters being added to --- tfact_grade (tracked in epic #2072). +-- semester and passing_grade are now available on dim_course_run (dimensional layer), +-- sourced via the micromasters_examruns join added in this PR. +-- Grade data and exam run metadata still sourced from intermediates pending MicroMasters +-- being added to tfact_grade (tracked in epic #2072). with micromasters_exam_grades as (select * from {{ ref("int__micromasters__dedp_proctored_exam_grades") }}), mitxonline_exam_grades as (select * from {{ ref("int__mitxonline__proctored_exam_grades") }}), - micromasters_users as (select * from {{ ref("int__micromasters__users") }}), - course_run_dim as ( - select courserun_readable_id, semester, passing_grade from {{ ref("dim_course_run") }} where is_current = true - ) + micromasters_users as (select * from {{ ref("int__micromasters__users") }}) select course_number, @@ -18,12 +16,11 @@ select user_full_name, user_micromasters_email, user_mitxonline_email, - course_run_dim.passing_grade as proctoredexamgrade_passing_grade, + micromasters_exam_grades.examrun_passing_grade as proctoredexamgrade_passing_grade, proctoredexamgrade_percentage_grade, proctoredexamgrade_created_on, - course_run_dim.semester + micromasters_exam_grades.examrun_semester as semester from micromasters_exam_grades -left join course_run_dim on micromasters_exam_grades.examrun_readable_id = course_run_dim.courserun_readable_id union all @@ -36,13 +33,12 @@ select mitxonline_exam_grades.user_full_name, micromasters_users.user_email as user_micromasters_email, mitxonline_exam_grades.user_email as user_mitxonline_email, - course_run_dim.passing_grade as proctoredexamgrade_passing_grade, + mitxonline_exam_grades.proctoredexamgrade_passing_grade, mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade, mitxonline_exam_grades.proctoredexamgrade_created_on, - course_run_dim.semester + mitxonline_exam_grades.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username -left join course_run_dim on mitxonline_exam_grades.courserun_readable_id = course_run_dim.courserun_readable_id left join micromasters_exam_grades on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id From 92d69c0a2fd9a5c546ab3edbdc1f3032a4ee24ce Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:18:22 -0400 Subject: [PATCH 04/11] revert: restore marts__micromasters_dedp_exam_grades to main (no functional changes in this PR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mart is unchanged from main — semester/passing_grade continue to source from int models. The dim_course_run additions in this PR are for future dimensional consumers; the mart migration is deferred to epic #2072 follow-up. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../marts__micromasters_dedp_exam_grades.sql | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 47e75397c..2a1190c53 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -1,46 +1,49 @@ --- semester and passing_grade are now available on dim_course_run (dimensional layer), --- sourced via the micromasters_examruns join added in this PR. --- Grade data and exam run metadata still sourced from intermediates pending MicroMasters --- being added to tfact_grade (tracked in epic #2072). -with - micromasters_exam_grades as (select * from {{ ref("int__micromasters__dedp_proctored_exam_grades") }}), - mitxonline_exam_grades as (select * from {{ ref("int__mitxonline__proctored_exam_grades") }}), - micromasters_users as (select * from {{ ref("int__micromasters__users") }}) +with micromasters_exam_grades as ( + select * from {{ ref('int__micromasters__dedp_proctored_exam_grades') }} +) + +, mitxonline_exam_grades as ( + select * from {{ ref('int__mitxonline__proctored_exam_grades') }} +) + +, micromasters_users as ( + select * from {{ ref('int__micromasters__users') }} +) select - course_number, - course_title, - micromasters_exam_grades.examrun_readable_id as examrun_courserun_readable_id, - user_edxorg_username, - user_mitxonline_username, - user_full_name, - user_micromasters_email, - user_mitxonline_email, - micromasters_exam_grades.examrun_passing_grade as proctoredexamgrade_passing_grade, - proctoredexamgrade_percentage_grade, - proctoredexamgrade_created_on, - micromasters_exam_grades.examrun_semester as semester + course_number + , course_title + , examrun_readable_id as examrun_courserun_readable_id + , user_edxorg_username + , user_mitxonline_username + , user_full_name + , user_micromasters_email + , user_mitxonline_email + , examrun_passing_grade as proctoredexamgrade_passing_grade + , proctoredexamgrade_percentage_grade + , proctoredexamgrade_created_on + , examrun_semester as semester from micromasters_exam_grades union all select - mitxonline_exam_grades.course_number, - mitxonline_exam_grades.course_title, - mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id, - mitxonline_exam_grades.user_edxorg_username, - mitxonline_exam_grades.user_username as user_mitxonline_username, - mitxonline_exam_grades.user_full_name, - micromasters_users.user_email as user_micromasters_email, - mitxonline_exam_grades.user_email as user_mitxonline_email, - mitxonline_exam_grades.proctoredexamgrade_passing_grade, - mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade, - mitxonline_exam_grades.proctoredexamgrade_created_on, - mitxonline_exam_grades.semester + mitxonline_exam_grades.course_number + , mitxonline_exam_grades.course_title + , mitxonline_exam_grades.courserun_readable_id as examrun_courserun_readable_id + , mitxonline_exam_grades.user_edxorg_username + , mitxonline_exam_grades.user_username as user_mitxonline_username + , mitxonline_exam_grades.user_full_name + , micromasters_users.user_email as user_micromasters_email + , mitxonline_exam_grades.user_email as user_mitxonline_email + , mitxonline_exam_grades.proctoredexamgrade_passing_grade + , mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade + , mitxonline_exam_grades.proctoredexamgrade_created_on + , mitxonline_exam_grades.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username -left join - micromasters_exam_grades - on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id - and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username +left join micromasters_exam_grades + on + mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id + and mitxonline_exam_grades.user_username = micromasters_exam_grades.user_mitxonline_username where micromasters_exam_grades.user_mitxonline_username is null and micromasters_exam_grades.examrun_readable_id is null From 96ee431f4b08f951697c37eea62683cac90ae307 Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Tue, 30 Jun 2026 17:04:51 -0400 Subject: [PATCH 05/11] fix(dim_course_run): dedup micromasters_examruns CTE; fix semester docs and null test - Add QUALIFY ROW_NUMBER() dedup to micromasters_examruns CTE to prevent 1:N fan-out if multiple exam run records share the same examrun_readable_id; picks the most recently updated row per readable ID. - Fix semester column description in _dim_course_run.yml: value format is '2T2022' (term identifier), not 'Fall 2023' (human-readable label). - Remove not_null test on semester in _marts_micromasters__models.yml: MITxOnline exam grade rows without a corresponding MicroMasters exam run record will have semester = NULL, so the constraint was incorrect. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../models/dimensional/_dim_course_run.yml | 2 +- .../models/dimensional/dim_course_run.sql | 558 +++++++++--------- .../_marts_micromasters__models.yml | 5 +- 3 files changed, 280 insertions(+), 285 deletions(-) diff --git a/src/ol_dbt/models/dimensional/_dim_course_run.yml b/src/ol_dbt/models/dimensional/_dim_course_run.yml index fb63791e3..b96f159ff 100644 --- a/src/ol_dbt/models/dimensional/_dim_course_run.yml +++ b/src/ol_dbt/models/dimensional/_dim_course_run.yml @@ -79,7 +79,7 @@ models: mitxonline, mitxpro, and residential platforms. Null for edxorg and bootcamps. - name: semester description: > - Academic term label for a MicroMasters proctored exam run (e.g. "Fall 2023"). + Academic term identifier for a MicroMasters proctored exam run (e.g. "2T2022"). Populated only for MITxOnline course runs that correspond to a MicroMasters exam run. Null for all other platforms and non-exam MITxOnline course runs. diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index 872b735aa..5bf46d90b 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -1,292 +1,288 @@ -{{ config( - materialized='incremental', - unique_key=['courserun_pk', 'effective_date'], - incremental_strategy='delete+insert', - on_schema_change='append_new_columns' -) }} +{{ + config( + materialized="incremental", + unique_key=["courserun_pk", "effective_date"], + incremental_strategy="delete+insert", + on_schema_change="append_new_columns", + ) +}} -- MicroMasters stores exam run metadata (semester label and passing grade threshold) -- keyed by examrun_readable_id, which matches the MITxOnline courserun_readable_id for -- proctored exam course runs. These are the only platform-specific dimensional attributes -- that originate outside the platform's own course run record. -with micromasters_examruns as ( - select examrun_readable_id, examrun_semester, examrun_passing_grade - from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} -) - -, mitxonline_courseruns as ( - select - cr.courserun_readable_id - , cr.courserun_id as source_id - , cr.course_id - , cr.courserun_title - , cr.courserun_start_on - , cr.courserun_end_on - , cr.courserun_enrollment_start_on as enrollment_start - , cr.courserun_enrollment_end_on as enrollment_end - , cr.courserun_is_live - , cr.courserun_created_on - , cast(null as varchar) as course_readable_id - , er.examrun_semester as semester - , er.examrun_passing_grade as passing_grade - , 'mitxonline' as platform - from {{ ref('int__mitxonline__course_runs') }} as cr - left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id -) - -, mitxpro_courseruns as ( - select - courserun_readable_id - , courserun_id as source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , courserun_is_live - , courserun_created_on - , cast(null as varchar) as course_readable_id - , cast(null as varchar) as semester - , cast(null as double) as passing_grade - , 'mitxpro' as platform - from {{ ref('int__mitxpro__course_runs') }} -) - -, edxorg_courseruns as ( - select - courserun_readable_id - , cast(null as integer) as source_id - , cast(null as integer) as course_id - , courserun_title - , courserun_start_date as courserun_start_on -- edxorg uses _date suffix - , courserun_end_date as courserun_end_on - , courserun_enrollment_start_date as enrollment_start - , courserun_enrollment_end_date as enrollment_end - , courserun_is_published as courserun_is_live - , cast(null as varchar) as courserun_created_on - , cast(null as varchar) as course_readable_id - , cast(null as varchar) as semester - , cast(null as double) as passing_grade - , 'edxorg' as platform - from {{ ref('int__edxorg__mitx_courseruns') }} -) - -, residential_courseruns as ( - select - courserun_readable_id - , cast(null as integer) as source_id - , cast(null as integer) as course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , courserun_enrollment_start_on as enrollment_start - , courserun_enrollment_end_on as enrollment_end - , cast(null as boolean) as courserun_is_live - , courserun_created_on - , cast(null as varchar) as course_readable_id - , cast(null as varchar) as semester - , cast(null as double) as passing_grade - , 'residential' as platform - from {{ ref('int__mitxresidential__courseruns') }} -) - -, bootcamps_courseruns as ( - select - runs.courserun_readable_id - , runs.courserun_id as source_id - , runs.course_id - , runs.courserun_title - , runs.courserun_start_on - , runs.courserun_end_on - , cast(null as varchar) as enrollment_start - , cast(null as varchar) as enrollment_end - , false as courserun_is_live - , cast(null as varchar) as courserun_created_on - , runs.course_readable_id - , cast(null as varchar) as semester - , cast(null as double) as passing_grade - , 'bootcamps' as platform - from {{ ref('int__bootcamps__course_runs') }} as runs -) - -, combined_courseruns as ( - select * from mitxonline_courseruns - union all - select * from mitxpro_courseruns - union all - select * from edxorg_courseruns - union all - select * from residential_courseruns - union all - select * from bootcamps_courseruns -) +with + micromasters_examruns as ( + select examrun_readable_id, examrun_semester, examrun_passing_grade + from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} + qualify row_number() over (partition by examrun_readable_id order by examrun_updated_on desc nulls last) = 1 + ), + mitxonline_courseruns as ( + select + cr.courserun_readable_id, + cr.courserun_id as source_id, + cr.course_id, + cr.courserun_title, + cr.courserun_start_on, + cr.courserun_end_on, + cr.courserun_enrollment_start_on as enrollment_start, + cr.courserun_enrollment_end_on as enrollment_end, + cr.courserun_is_live, + cr.courserun_created_on, + cast(null as varchar) as course_readable_id, + er.examrun_semester as semester, + er.examrun_passing_grade as passing_grade, + 'mitxonline' as platform + from {{ ref("int__mitxonline__course_runs") }} as cr + left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id + ), + mitxpro_courseruns as ( + select + courserun_readable_id, + courserun_id as source_id, + course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + courserun_enrollment_start_on as enrollment_start, + courserun_enrollment_end_on as enrollment_end, + courserun_is_live, + courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'mitxpro' as platform + from {{ ref("int__mitxpro__course_runs") }} + ), + edxorg_courseruns as ( + select + courserun_readable_id, + cast(null as integer) as source_id, + cast(null as integer) as course_id, + courserun_title, + courserun_start_date as courserun_start_on, -- edxorg uses _date suffix + courserun_end_date as courserun_end_on, + courserun_enrollment_start_date as enrollment_start, + courserun_enrollment_end_date as enrollment_end, + courserun_is_published as courserun_is_live, + cast(null as varchar) as courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'edxorg' as platform + from {{ ref("int__edxorg__mitx_courseruns") }} + ), + residential_courseruns as ( + select + courserun_readable_id, + cast(null as integer) as source_id, + cast(null as integer) as course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + courserun_enrollment_start_on as enrollment_start, + courserun_enrollment_end_on as enrollment_end, + cast(null as boolean) as courserun_is_live, + courserun_created_on, + cast(null as varchar) as course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'residential' as platform + from {{ ref("int__mitxresidential__courseruns") }} + ), + bootcamps_courseruns as ( + select + runs.courserun_readable_id, + runs.courserun_id as source_id, + runs.course_id, + runs.courserun_title, + runs.courserun_start_on, + runs.courserun_end_on, + cast(null as varchar) as enrollment_start, + cast(null as varchar) as enrollment_end, + false as courserun_is_live, + cast(null as varchar) as courserun_created_on, + runs.course_readable_id, + cast(null as varchar) as semester, + cast(null as double) as passing_grade, + 'bootcamps' as platform + from {{ ref("int__bootcamps__course_runs") }} as runs + ), + combined_courseruns as ( + select * + from mitxonline_courseruns + union all + select * + from mitxpro_courseruns + union all + select * + from edxorg_courseruns + union all + select * + from residential_courseruns + union all + select * + from bootcamps_courseruns + ), + -- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality + combined_courseruns_resolved as ( + select + courserun_readable_id, + source_id, + course_id, + courserun_title, + courserun_start_on, + courserun_end_on, + enrollment_start, + enrollment_end, + courserun_is_live, + courserun_created_on, + platform, + semester, + passing_grade, + coalesce( + course_readable_id, + case + when courserun_readable_id like 'course-v1:%' + then + substring( + courserun_readable_id, + 1, + length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '+') + ) + when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') + then + substring( + courserun_readable_id, + 1, + length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '/') + ) + else courserun_readable_id + end + ) as course_readable_id + from combined_courseruns + ), + -- Join to dim_course to get course_fk + dim_course as ( + select course_pk, course_readable_id, primary_platform from {{ ref("dim_course") }} where is_current = true + ), + courseruns_with_fk as ( + select combined_courseruns_resolved.*, dim_course.course_pk as course_fk + from combined_courseruns_resolved + left join + dim_course + on combined_courseruns_resolved.platform = dim_course.primary_platform + and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id + ), + courseruns_with_all_fks as ( + select + courseruns_with_fk.*, + dim_platform_lookup.platform_pk as platform_fk, + -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) + {{ iso8601_to_date_key("courserun_start_on") }} as courserun_start_date_key, + {{ iso8601_to_date_key("courserun_end_on") }} as courserun_end_date_key, + {{ iso8601_to_date_key("enrollment_start") }} as enrollment_start_date_key, + {{ iso8601_to_date_key("enrollment_end") }} as enrollment_end_date_key + from courseruns_with_fk + left join + (select platform_pk, platform_readable_id from {{ ref("dim_platform") }}) as dim_platform_lookup + on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id + ), + final as ( + select + {{ dbt_utils.generate_surrogate_key(["platform", "courserun_readable_id"]) }} as courserun_pk, + courserun_readable_id, + source_id, + course_fk, + platform_fk, + platform, + courserun_title, + courserun_start_date_key, + courserun_end_date_key, + enrollment_start_date_key, + enrollment_end_date_key, + courserun_start_on, + courserun_end_on, + enrollment_start, + enrollment_end, + courserun_is_live, + courserun_created_on, + semester, + passing_grade, + current_timestamp as effective_date, + cast(null as timestamp) as end_date, + true as is_current + from courseruns_with_all_fks --- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality -, combined_courseruns_resolved as ( - select - courserun_readable_id - , source_id - , course_id - , courserun_title - , courserun_start_on - , courserun_end_on - , enrollment_start - , enrollment_end - , courserun_is_live - , courserun_created_on - , platform - , semester - , passing_grade - , coalesce( - course_readable_id, - case - when courserun_readable_id like 'course-v1:%' - then substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - - strpos(reverse(courserun_readable_id), '+') - ) - when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') - then substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - - strpos(reverse(courserun_readable_id), '/') - ) - else courserun_readable_id - end - ) as course_readable_id - from combined_courseruns -) - --- Join to dim_course to get course_fk -, dim_course as ( - select - course_pk - , course_readable_id - , primary_platform - from {{ ref('dim_course') }} - where is_current = true -) - -, courseruns_with_fk as ( - select - combined_courseruns_resolved.* - , dim_course.course_pk as course_fk - from combined_courseruns_resolved - left join dim_course - on combined_courseruns_resolved.platform = dim_course.primary_platform - and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id -) - -, courseruns_with_all_fks as ( - select - courseruns_with_fk.* - , dim_platform_lookup.platform_pk as platform_fk - -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) - , {{ iso8601_to_date_key('courserun_start_on') }} as courserun_start_date_key - , {{ iso8601_to_date_key('courserun_end_on') }} as courserun_end_date_key - , {{ iso8601_to_date_key('enrollment_start') }} as enrollment_start_date_key - , {{ iso8601_to_date_key('enrollment_end') }} as enrollment_end_date_key - from courseruns_with_fk - left join ( - select platform_pk, platform_readable_id - from {{ ref('dim_platform') }} - ) as dim_platform_lookup - on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id -) - -, final as ( - select - {{ dbt_utils.generate_surrogate_key([ - 'platform', - 'courserun_readable_id' - ]) }} as courserun_pk - , courserun_readable_id - , source_id - , course_fk - , platform_fk - , platform - , courserun_title - , courserun_start_date_key - , courserun_end_date_key - , enrollment_start_date_key - , enrollment_end_date_key - , courserun_start_on - , courserun_end_on - , enrollment_start - , enrollment_end - , courserun_is_live - , courserun_created_on - , semester - , passing_grade - , current_timestamp as effective_date - , cast(null as timestamp) as end_date - , true as is_current - from courseruns_with_all_fks - - {% if is_incremental() %} - where not exists ( - select 1 - from {{ this }} as existing - where - existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id - and existing.platform = courseruns_with_all_fks.platform - and existing.is_current = true - and existing.courserun_title = courseruns_with_all_fks.courserun_title - and coalesce(existing.courserun_start_on, '') = coalesce(courseruns_with_all_fks.courserun_start_on, '') - and coalesce(existing.courserun_end_on, '') = coalesce(courseruns_with_all_fks.courserun_end_on, '') - and coalesce(existing.enrollment_start, '') = coalesce(courseruns_with_all_fks.enrollment_start, '') - and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') - and coalesce(existing.courserun_is_live, false) = coalesce(courseruns_with_all_fks.courserun_is_live, false) - and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') - and coalesce(existing.passing_grade, -1.0) = coalesce(courseruns_with_all_fks.passing_grade, -1.0) + {% if is_incremental() %} + where + not exists ( + select 1 + from {{ this }} as existing + where + existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id + and existing.platform = courseruns_with_all_fks.platform + and existing.is_current = true + and existing.courserun_title = courseruns_with_all_fks.courserun_title + and coalesce(existing.courserun_start_on, '') + = coalesce(courseruns_with_all_fks.courserun_start_on, '') + and coalesce(existing.courserun_end_on, '') + = coalesce(courseruns_with_all_fks.courserun_end_on, '') + and coalesce(existing.enrollment_start, '') + = coalesce(courseruns_with_all_fks.enrollment_start, '') + and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') + and coalesce(existing.courserun_is_live, false) + = coalesce(courseruns_with_all_fks.courserun_is_live, false) + and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') + and coalesce(existing.passing_grade, -1.0) + = coalesce(courseruns_with_all_fks.passing_grade, -1.0) + ) + {% endif %} ) - {% endif %} -) {% if is_incremental() %} --- Expire prior current rows that have changed -, records_to_expire as ( - select - existing.courserun_pk - , existing.courserun_readable_id - , existing.source_id - , existing.course_fk - , existing.platform_fk - , existing.platform - , existing.courserun_title - , existing.courserun_start_date_key - , existing.courserun_end_date_key - , existing.enrollment_start_date_key - , existing.enrollment_end_date_key - , existing.courserun_start_on - , existing.courserun_end_on - , existing.enrollment_start - , existing.enrollment_end - , existing.courserun_is_live - , existing.courserun_created_on - , existing.semester - , existing.passing_grade - , existing.effective_date - , current_timestamp as end_date - , false as is_current - from {{ this }} as existing - inner join final as new_records - on existing.courserun_readable_id = new_records.courserun_readable_id - and existing.platform = new_records.platform - where existing.is_current = true -) - -, combined as ( - select * from final - union all - select * from records_to_expire -) + , + -- Expire prior current rows that have changed + records_to_expire as ( + select + existing.courserun_pk, + existing.courserun_readable_id, + existing.source_id, + existing.course_fk, + existing.platform_fk, + existing.platform, + existing.courserun_title, + existing.courserun_start_date_key, + existing.courserun_end_date_key, + existing.enrollment_start_date_key, + existing.enrollment_end_date_key, + existing.courserun_start_on, + existing.courserun_end_on, + existing.enrollment_start, + existing.enrollment_end, + existing.courserun_is_live, + existing.courserun_created_on, + existing.semester, + existing.passing_grade, + existing.effective_date, + current_timestamp as end_date, + false as is_current + from {{ this }} as existing + inner join + final as new_records + on existing.courserun_readable_id = new_records.courserun_readable_id + and existing.platform = new_records.platform + where existing.is_current = true + ), + combined as ( + select * + from final + union all + select * + from records_to_expire + ) -select * from combined -{% else %} -select * from final + select * + from combined +{% else %} select * from final {% endif %} diff --git a/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml b/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml index 8e8c476a4..69f9c589e 100644 --- a/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml +++ b/src/ol_dbt/models/marts/micromasters/_marts_micromasters__models.yml @@ -39,9 +39,8 @@ models: tests: - not_null - name: semester - description: str, semester for the exam, e.g., 2T2022. - tests: - - not_null + description: str, semester for the exam, e.g., 2T2022. Null for MITx Online exam + runs that do not have a corresponding MicroMasters exam run record. tests: - dbt_expectations.expect_compound_columns_to_be_unique: arguments: From 39dec9a1e2c0d6020c24df24140f6909810a607e Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:55:40 -0400 Subject: [PATCH 06/11] fix(dim_course_run): replace QUALIFY with ROW_NUMBER subquery for Trino compatibility Trino does not support the QUALIFY clause. Replace the micromasters_examruns dedup with the equivalent ROW_NUMBER() inner-subquery pattern, consistent with the existing pattern documented and used in dim_course.sql. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ol_dbt/models/dimensional/dim_course_run.sql | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index 5bf46d90b..b95262674 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -14,8 +14,18 @@ with micromasters_examruns as ( select examrun_readable_id, examrun_semester, examrun_passing_grade - from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} - qualify row_number() over (partition by examrun_readable_id order by examrun_updated_on desc nulls last) = 1 + from + ( + select + examrun_readable_id, + examrun_semester, + examrun_passing_grade, + row_number() over ( + partition by examrun_readable_id order by examrun_updated_on desc nulls last + ) as _row_num + from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} + ) + where _row_num = 1 ), mitxonline_courseruns as ( select From fe7303dabe47ecd91b62cb16bc8c83c749a89069 Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:06:02 -0400 Subject: [PATCH 07/11] fix: add deterministic tie-breaker to micromasters_examruns dedup Add examrun_id desc as secondary ORDER BY key so the ROW_NUMBER dedup is deterministic when two exam run rows share the same examrun_updated_on timestamp, preventing nondeterministic semester/passing_grade selection and unnecessary SCD2 churn. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ol_dbt/models/dimensional/dim_course_run.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index b95262674..1fba3576e 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -21,7 +21,7 @@ with examrun_semester, examrun_passing_grade, row_number() over ( - partition by examrun_readable_id order by examrun_updated_on desc nulls last + partition by examrun_readable_id order by examrun_updated_on desc nulls last, examrun_id desc ) as _row_num from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} ) From 6dfc8dd062af5afa97d056c8edada1b0fd1dd829 Mon Sep 17 00:00:00 2001 From: Quazi <59845076+quazi-h@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:08:58 -0400 Subject: [PATCH 08/11] style: split else branch onto separate line for consistency Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ol_dbt/models/dimensional/dim_course_run.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index 1fba3576e..c1e23e005 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -294,5 +294,7 @@ with select * from combined -{% else %} select * from final +{% else %} + select * + from final {% endif %} From 79c5b474b6f7f7f4b09a5fc4c7b6a3b16d773942 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:55:31 -0400 Subject: [PATCH 09/11] fix(dim_course_run): alias micromasters_examruns dedup subquery for Trino Trino requires derived tables in a FROM clause to be aliased. Adds alias to the row_number() dedup subquery per review feedback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ol_dbt/models/dimensional/dim_course_run.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index c1e23e005..49a2ef1f0 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -24,7 +24,7 @@ with partition by examrun_readable_id order by examrun_updated_on desc nulls last, examrun_id desc ) as _row_num from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} - ) + ) as deduped_examruns where _row_num = 1 ), mitxonline_courseruns as ( From 1f3095dd21be192dc69a8cbe231707003c8764e0 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:30:28 -0400 Subject: [PATCH 10/11] style(dim_course_run): split row_number window clause across lines Matches formatting convention in dim_course.sql per review feedback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ol_dbt/models/dimensional/dim_course_run.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index 49a2ef1f0..4552dee46 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -21,7 +21,8 @@ with examrun_semester, examrun_passing_grade, row_number() over ( - partition by examrun_readable_id order by examrun_updated_on desc nulls last, examrun_id desc + partition by examrun_readable_id + order by examrun_updated_on desc nulls last, examrun_id desc ) as _row_num from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} ) as deduped_examruns From 313f372546dfb953cc0ad777040cc5b37221150f Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:11:46 -0400 Subject: [PATCH 11/11] style(dim_course_run): drop incidental reformat, keep original formatting Reconstructs dim_course_run.sql to preserve the file's original leading-comma formatting instead of the full-file trailing-comma rewrite introduced earlier in this PR. Only the actual logical additions from #2088 are retained: - new micromasters_examruns CTE (deduped via row_number, keyed on examrun_readable_id) - semester/passing_grade threaded through the mitxonline_courseruns join, all other per-platform CTEs (as NULL), combined_courseruns_resolved, the final SELECT, the SCD2 change-detection predicate, and records_to_expire Verified equivalent to the prior version via token-level diff (only a trailing-comma-in-config difference remains), dbt compile against a local duckdb target, and sqlfluff-lint/-fix (no changes needed). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../models/dimensional/dim_course_run.sql | 579 +++++++++--------- 1 file changed, 291 insertions(+), 288 deletions(-) diff --git a/src/ol_dbt/models/dimensional/dim_course_run.sql b/src/ol_dbt/models/dimensional/dim_course_run.sql index 4552dee46..a28f9b041 100644 --- a/src/ol_dbt/models/dimensional/dim_course_run.sql +++ b/src/ol_dbt/models/dimensional/dim_course_run.sql @@ -1,301 +1,304 @@ -{{ - config( - materialized="incremental", - unique_key=["courserun_pk", "effective_date"], - incremental_strategy="delete+insert", - on_schema_change="append_new_columns", - ) -}} +{{ config( + materialized='incremental', + unique_key=['courserun_pk', 'effective_date'], + incremental_strategy='delete+insert', + on_schema_change='append_new_columns' +) }} -- MicroMasters stores exam run metadata (semester label and passing grade threshold) -- keyed by examrun_readable_id, which matches the MITxOnline courserun_readable_id for -- proctored exam course runs. These are the only platform-specific dimensional attributes -- that originate outside the platform's own course run record. -with - micromasters_examruns as ( - select examrun_readable_id, examrun_semester, examrun_passing_grade - from - ( - select - examrun_readable_id, - examrun_semester, - examrun_passing_grade, - row_number() over ( - partition by examrun_readable_id - order by examrun_updated_on desc nulls last, examrun_id desc - ) as _row_num - from {{ ref("stg__micromasters__app__postgres__exams_examrun") }} - ) as deduped_examruns - where _row_num = 1 - ), - mitxonline_courseruns as ( - select - cr.courserun_readable_id, - cr.courserun_id as source_id, - cr.course_id, - cr.courserun_title, - cr.courserun_start_on, - cr.courserun_end_on, - cr.courserun_enrollment_start_on as enrollment_start, - cr.courserun_enrollment_end_on as enrollment_end, - cr.courserun_is_live, - cr.courserun_created_on, - cast(null as varchar) as course_readable_id, - er.examrun_semester as semester, - er.examrun_passing_grade as passing_grade, - 'mitxonline' as platform - from {{ ref("int__mitxonline__course_runs") }} as cr - left join micromasters_examruns as er on cr.courserun_readable_id = er.examrun_readable_id - ), - mitxpro_courseruns as ( - select - courserun_readable_id, - courserun_id as source_id, - course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - courserun_enrollment_start_on as enrollment_start, - courserun_enrollment_end_on as enrollment_end, - courserun_is_live, - courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'mitxpro' as platform - from {{ ref("int__mitxpro__course_runs") }} - ), - edxorg_courseruns as ( - select - courserun_readable_id, - cast(null as integer) as source_id, - cast(null as integer) as course_id, - courserun_title, - courserun_start_date as courserun_start_on, -- edxorg uses _date suffix - courserun_end_date as courserun_end_on, - courserun_enrollment_start_date as enrollment_start, - courserun_enrollment_end_date as enrollment_end, - courserun_is_published as courserun_is_live, - cast(null as varchar) as courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'edxorg' as platform - from {{ ref("int__edxorg__mitx_courseruns") }} - ), - residential_courseruns as ( - select - courserun_readable_id, - cast(null as integer) as source_id, - cast(null as integer) as course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - courserun_enrollment_start_on as enrollment_start, - courserun_enrollment_end_on as enrollment_end, - cast(null as boolean) as courserun_is_live, - courserun_created_on, - cast(null as varchar) as course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'residential' as platform - from {{ ref("int__mitxresidential__courseruns") }} - ), - bootcamps_courseruns as ( +with micromasters_examruns as ( + select examrun_readable_id, examrun_semester, examrun_passing_grade + from ( select - runs.courserun_readable_id, - runs.courserun_id as source_id, - runs.course_id, - runs.courserun_title, - runs.courserun_start_on, - runs.courserun_end_on, - cast(null as varchar) as enrollment_start, - cast(null as varchar) as enrollment_end, - false as courserun_is_live, - cast(null as varchar) as courserun_created_on, - runs.course_readable_id, - cast(null as varchar) as semester, - cast(null as double) as passing_grade, - 'bootcamps' as platform - from {{ ref("int__bootcamps__course_runs") }} as runs - ), - combined_courseruns as ( - select * - from mitxonline_courseruns - union all - select * - from mitxpro_courseruns - union all - select * - from edxorg_courseruns - union all - select * - from residential_courseruns - union all - select * - from bootcamps_courseruns - ), - -- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality - combined_courseruns_resolved as ( - select - courserun_readable_id, - source_id, - course_id, - courserun_title, - courserun_start_on, - courserun_end_on, - enrollment_start, - enrollment_end, - courserun_is_live, - courserun_created_on, - platform, - semester, - passing_grade, - coalesce( - course_readable_id, - case - when courserun_readable_id like 'course-v1:%' - then - substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '+') - ) - when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') - then - substring( - courserun_readable_id, - 1, - length(courserun_readable_id) - strpos(reverse(courserun_readable_id), '/') - ) - else courserun_readable_id - end - ) as course_readable_id - from combined_courseruns - ), - -- Join to dim_course to get course_fk - dim_course as ( - select course_pk, course_readable_id, primary_platform from {{ ref("dim_course") }} where is_current = true - ), - courseruns_with_fk as ( - select combined_courseruns_resolved.*, dim_course.course_pk as course_fk - from combined_courseruns_resolved - left join - dim_course - on combined_courseruns_resolved.platform = dim_course.primary_platform - and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id - ), - courseruns_with_all_fks as ( - select - courseruns_with_fk.*, - dim_platform_lookup.platform_pk as platform_fk, - -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) - {{ iso8601_to_date_key("courserun_start_on") }} as courserun_start_date_key, - {{ iso8601_to_date_key("courserun_end_on") }} as courserun_end_date_key, - {{ iso8601_to_date_key("enrollment_start") }} as enrollment_start_date_key, - {{ iso8601_to_date_key("enrollment_end") }} as enrollment_end_date_key - from courseruns_with_fk - left join - (select platform_pk, platform_readable_id from {{ ref("dim_platform") }}) as dim_platform_lookup - on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id - ), - final as ( - select - {{ dbt_utils.generate_surrogate_key(["platform", "courserun_readable_id"]) }} as courserun_pk, - courserun_readable_id, - source_id, - course_fk, - platform_fk, - platform, - courserun_title, - courserun_start_date_key, - courserun_end_date_key, - enrollment_start_date_key, - enrollment_end_date_key, - courserun_start_on, - courserun_end_on, - enrollment_start, - enrollment_end, - courserun_is_live, - courserun_created_on, - semester, - passing_grade, - current_timestamp as effective_date, - cast(null as timestamp) as end_date, - true as is_current - from courseruns_with_all_fks + examrun_readable_id + , examrun_semester + , examrun_passing_grade + , row_number() over ( + partition by examrun_readable_id + order by examrun_updated_on desc nulls last, examrun_id desc + ) as _row_num + from {{ ref('stg__micromasters__app__postgres__exams_examrun') }} + ) as deduped_examruns + where _row_num = 1 +) + +, mitxonline_courseruns as ( + select + cr.courserun_readable_id + , cr.courserun_id as source_id + , cr.course_id + , cr.courserun_title + , cr.courserun_start_on + , cr.courserun_end_on + , cr.courserun_enrollment_start_on as enrollment_start + , cr.courserun_enrollment_end_on as enrollment_end + , cr.courserun_is_live + , cr.courserun_created_on + , cast(null as varchar) as course_readable_id + , er.examrun_semester as semester + , er.examrun_passing_grade as passing_grade + , 'mitxonline' as platform + from {{ ref('int__mitxonline__course_runs') }} as cr + left join micromasters_examruns as er + on cr.courserun_readable_id = er.examrun_readable_id +) + +, mitxpro_courseruns as ( + select + courserun_readable_id + , courserun_id as source_id + , course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , courserun_enrollment_start_on as enrollment_start + , courserun_enrollment_end_on as enrollment_end + , courserun_is_live + , courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'mitxpro' as platform + from {{ ref('int__mitxpro__course_runs') }} +) - {% if is_incremental() %} - where - not exists ( - select 1 - from {{ this }} as existing - where - existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id - and existing.platform = courseruns_with_all_fks.platform - and existing.is_current = true - and existing.courserun_title = courseruns_with_all_fks.courserun_title - and coalesce(existing.courserun_start_on, '') - = coalesce(courseruns_with_all_fks.courserun_start_on, '') - and coalesce(existing.courserun_end_on, '') - = coalesce(courseruns_with_all_fks.courserun_end_on, '') - and coalesce(existing.enrollment_start, '') - = coalesce(courseruns_with_all_fks.enrollment_start, '') - and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') - and coalesce(existing.courserun_is_live, false) - = coalesce(courseruns_with_all_fks.courserun_is_live, false) - and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') - and coalesce(existing.passing_grade, -1.0) - = coalesce(courseruns_with_all_fks.passing_grade, -1.0) - ) - {% endif %} +, edxorg_courseruns as ( + select + courserun_readable_id + , cast(null as integer) as source_id + , cast(null as integer) as course_id + , courserun_title + , courserun_start_date as courserun_start_on -- edxorg uses _date suffix + , courserun_end_date as courserun_end_on + , courserun_enrollment_start_date as enrollment_start + , courserun_enrollment_end_date as enrollment_end + , courserun_is_published as courserun_is_live + , cast(null as varchar) as courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'edxorg' as platform + from {{ ref('int__edxorg__mitx_courseruns') }} +) + +, residential_courseruns as ( + select + courserun_readable_id + , cast(null as integer) as source_id + , cast(null as integer) as course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , courserun_enrollment_start_on as enrollment_start + , courserun_enrollment_end_on as enrollment_end + , cast(null as boolean) as courserun_is_live + , courserun_created_on + , cast(null as varchar) as course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'residential' as platform + from {{ ref('int__mitxresidential__courseruns') }} +) + +, bootcamps_courseruns as ( + select + runs.courserun_readable_id + , runs.courserun_id as source_id + , runs.course_id + , runs.courserun_title + , runs.courserun_start_on + , runs.courserun_end_on + , cast(null as varchar) as enrollment_start + , cast(null as varchar) as enrollment_end + , false as courserun_is_live + , cast(null as varchar) as courserun_created_on + , runs.course_readable_id + , cast(null as varchar) as semester + , cast(null as double) as passing_grade + , 'bootcamps' as platform + from {{ ref('int__bootcamps__course_runs') }} as runs +) + +, combined_courseruns as ( + select * from mitxonline_courseruns + union all + select * from mitxpro_courseruns + union all + select * from edxorg_courseruns + union all + select * from residential_courseruns + union all + select * from bootcamps_courseruns +) + +-- Pre-compute course_readable_id for all platforms so the dim_course join is a simple equality +, combined_courseruns_resolved as ( + select + courserun_readable_id + , source_id + , course_id + , courserun_title + , courserun_start_on + , courserun_end_on + , enrollment_start + , enrollment_end + , courserun_is_live + , courserun_created_on + , platform + , semester + , passing_grade + , coalesce( + course_readable_id, + case + when courserun_readable_id like 'course-v1:%' + then substring( + courserun_readable_id, + 1, + length(courserun_readable_id) + - strpos(reverse(courserun_readable_id), '+') + ) + when regexp_like(courserun_readable_id, '^[^/]+/[^/]+/[^/]+') + then substring( + courserun_readable_id, + 1, + length(courserun_readable_id) + - strpos(reverse(courserun_readable_id), '/') + ) + else courserun_readable_id + end + ) as course_readable_id + from combined_courseruns +) + +-- Join to dim_course to get course_fk +, dim_course as ( + select + course_pk + , course_readable_id + , primary_platform + from {{ ref('dim_course') }} + where is_current = true +) + +, courseruns_with_fk as ( + select + combined_courseruns_resolved.* + , dim_course.course_pk as course_fk + from combined_courseruns_resolved + left join dim_course + on combined_courseruns_resolved.platform = dim_course.primary_platform + and combined_courseruns_resolved.course_readable_id = dim_course.course_readable_id +) + +, courseruns_with_all_fks as ( + select + courseruns_with_fk.* + , dim_platform_lookup.platform_pk as platform_fk + -- Create date keys for date dimension joins (parse to timestamp then format as YYYYMMDD) + , {{ iso8601_to_date_key('courserun_start_on') }} as courserun_start_date_key + , {{ iso8601_to_date_key('courserun_end_on') }} as courserun_end_date_key + , {{ iso8601_to_date_key('enrollment_start') }} as enrollment_start_date_key + , {{ iso8601_to_date_key('enrollment_end') }} as enrollment_end_date_key + from courseruns_with_fk + left join ( + select platform_pk, platform_readable_id + from {{ ref('dim_platform') }} + ) as dim_platform_lookup + on courseruns_with_fk.platform = dim_platform_lookup.platform_readable_id +) + +, final as ( + select + {{ dbt_utils.generate_surrogate_key([ + 'platform', + 'courserun_readable_id' + ]) }} as courserun_pk + , courserun_readable_id + , source_id + , course_fk + , platform_fk + , platform + , courserun_title + , courserun_start_date_key + , courserun_end_date_key + , enrollment_start_date_key + , enrollment_end_date_key + , courserun_start_on + , courserun_end_on + , enrollment_start + , enrollment_end + , courserun_is_live + , courserun_created_on + , semester + , passing_grade + , current_timestamp as effective_date + , cast(null as timestamp) as end_date + , true as is_current + from courseruns_with_all_fks + + {% if is_incremental() %} + where not exists ( + select 1 + from {{ this }} as existing + where + existing.courserun_readable_id = courseruns_with_all_fks.courserun_readable_id + and existing.platform = courseruns_with_all_fks.platform + and existing.is_current = true + and existing.courserun_title = courseruns_with_all_fks.courserun_title + and coalesce(existing.courserun_start_on, '') = coalesce(courseruns_with_all_fks.courserun_start_on, '') + and coalesce(existing.courserun_end_on, '') = coalesce(courseruns_with_all_fks.courserun_end_on, '') + and coalesce(existing.enrollment_start, '') = coalesce(courseruns_with_all_fks.enrollment_start, '') + and coalesce(existing.enrollment_end, '') = coalesce(courseruns_with_all_fks.enrollment_end, '') + and coalesce(existing.courserun_is_live, false) = coalesce(courseruns_with_all_fks.courserun_is_live, false) + and coalesce(existing.semester, '') = coalesce(courseruns_with_all_fks.semester, '') + and coalesce(existing.passing_grade, -1.0) = coalesce(courseruns_with_all_fks.passing_grade, -1.0) ) + {% endif %} +) {% if is_incremental() %} - , - -- Expire prior current rows that have changed - records_to_expire as ( - select - existing.courserun_pk, - existing.courserun_readable_id, - existing.source_id, - existing.course_fk, - existing.platform_fk, - existing.platform, - existing.courserun_title, - existing.courserun_start_date_key, - existing.courserun_end_date_key, - existing.enrollment_start_date_key, - existing.enrollment_end_date_key, - existing.courserun_start_on, - existing.courserun_end_on, - existing.enrollment_start, - existing.enrollment_end, - existing.courserun_is_live, - existing.courserun_created_on, - existing.semester, - existing.passing_grade, - existing.effective_date, - current_timestamp as end_date, - false as is_current - from {{ this }} as existing - inner join - final as new_records - on existing.courserun_readable_id = new_records.courserun_readable_id - and existing.platform = new_records.platform - where existing.is_current = true - ), - combined as ( - select * - from final - union all - select * - from records_to_expire - ) +-- Expire prior current rows that have changed +, records_to_expire as ( + select + existing.courserun_pk + , existing.courserun_readable_id + , existing.source_id + , existing.course_fk + , existing.platform_fk + , existing.platform + , existing.courserun_title + , existing.courserun_start_date_key + , existing.courserun_end_date_key + , existing.enrollment_start_date_key + , existing.enrollment_end_date_key + , existing.courserun_start_on + , existing.courserun_end_on + , existing.enrollment_start + , existing.enrollment_end + , existing.courserun_is_live + , existing.courserun_created_on + , existing.semester + , existing.passing_grade + , existing.effective_date + , current_timestamp as end_date + , false as is_current + from {{ this }} as existing + inner join final as new_records + on existing.courserun_readable_id = new_records.courserun_readable_id + and existing.platform = new_records.platform + where existing.is_current = true +) + +, combined as ( + select * from final + union all + select * from records_to_expire +) - select * - from combined +select * from combined {% else %} - select * - from final +select * from final {% endif %}