feat(warehouses/accelerator): Mean time between failures model

infra/ansible-docker/playbooks/elt/group_vars/elt.yml

@@ -28,7 +28,7 @@ elt_warehouses:
       - name: opralogweb
         hour: 2
         minute: 11
-        dbt_args: "--select '+models/operations/mcr_equipment_downtime_records.sql'"
+        dbt_args: "--select '+models/operations' --exclude '+models/operations/power_consumption.sql'"
       - name: electricity_sharepoint
         hour: "*"
         minute: 5

infra/ansible-docker/playbooks/elt/templates/secrets/envvars.j2

@@ -18,8 +18,11 @@ DESTINATION__PYICEBERG__CREDENTIALS__CLIENT_ID={{ vault_ingestion_catalog_client
 DESTINATION__PYICEBERG__CREDENTIALS__CLIENT_SECRET={{ vault_ingestion_catalog_client_secret }}
 DESTINATION__PYICEBERG__CREDENTIALS__SCOPE=lakekeeper
 
-DBT_TRINO_USER={{ trino_ingestion_username }}
-DBT_TRINO_PASSWORD={{ vault_trino_ingestion_passwd }}
+DBT_TRINO_HTTP_SCHEME=https
 DBT_TRINO_HOST={{ top_level_domain }}
 DBT_TRINO_PORT={{ trino_https_port }}
+DBT_TRINO_USER={{ trino_ingestion_username }}
+DBT_TRINO_PASSWORD={{ vault_trino_ingestion_passwd }}
 DBT_TRINO_THREADS={{ elt_threads }}
+DBT_TRINO_CATALOG={{ warehouse.name }}
+DBT_TRINO_CATALOG_SCHEMA_PREFIX=

warehouses/accelerator/transform/accelerator/dbt_project.yml

@@ -17,20 +17,18 @@ clean-targets: # directories to be removed by `dbt clean`
   - "target"
   - "dbt_packages"
 
-vars:
-  # The start date of equipment downtime records from Opralog
-  opralog_epoch_date: "2017-04-25"
-
 models:
   +file_format: parquet
-  +materialized: table
+  +materialized: view
   accelerator:
     # The full schema name of the final tables is "{target_schema}_"{custom_schema}"
     #   - 'target_schema' is defined in profiles.yml
     #   - 'custom_schema' is defined for each collection of models below
     # See https://docs.getdbt.com/docs/build/custom-schemas
     staging:
       +schema: staging
-      +materialized: view
+    intermediate:
+      +schema: intermediate
     operations:
       +schema: operations
+      +materialized: table

warehouses/accelerator/transform/accelerator/macros/parse_utc_timestamp.sql

@@ -1,9 +1,16 @@
 -- Parse separate date and time strings into a single UTC timestamp
-{%- macro parse_utc_timestamp(date_col, date_format, time_col, time_format='HH:mm:ss') -%}
-  {{ return(adapter.dispatch('parse_utc_timestamp')(date_col, date_format, time_col, time_format)) }}
+{%- macro parse_utc_timestamp(date_col, date_format, time_col, time_format='HH:mm:ss', src_timezone='UTC') -%}
+  {{ return(adapter.dispatch('parse_utc_timestamp')(date_col, date_format, time_col, time_format, src_timezone)) }}
 {% endmacro %}
 
 
-{%- macro trino__parse_utc_timestamp(date_col, date_format, time_col, time_format) -%}
-  cast(parse_datetime({{ adapter.quote(date_col) }} || ' ' || {{ adapter.quote(time_col) }}, '{{ date_format ~ ' ' ~ time_format }}') as timestamp(6)) at time zone 'UTC'
+{%- macro trino__parse_utc_timestamp(date_col, date_format, time_col, time_format, src_timezone) -%}
+  with_timezone(
+    cast(
+      parse_datetime({{ adapter.quote(date_col) }} || ' ' || {{ adapter.quote(time_col) }},
+                     '{{ date_format ~ ' ' ~ time_format }}')
+                     as timestamp(6)
+        ),
+    '{{ src_timezone }}'
+  ) at time zone 'UTC'
 {%- endmacro -%}

warehouses/accelerator/transform/accelerator/models/operations/cycles.sql → warehouses/accelerator/transform/accelerator/models/intermediate/int_cycle_phases_without_target.sql

@@ -15,13 +15,14 @@ staged as (
 staged_discard_target as (
 
   select
-    name,
+
+    {{ adapter.quote('name') }},
     started_at,
     ended_at,
     phase
 
   from staged
-  group by name, started_at, ended_at, phase
+  group by {{ adapter.quote('name') }}, started_at, ended_at, phase
 
 )
 

warehouses/accelerator/transform/accelerator/models/intermediate/int_cycle_phases_without_target.yml

@@ -0,0 +1,20 @@
+models:
+  - name: int_cycle_phases_without_target
+    description: >
+      Each record defines a phase of an ISIS cycle, disregarding any target information
+    columns:
+      - name: name
+        data_tests:
+          - not_null
+          - unique
+      - name: phase
+        data_tests:
+          - accepted_values:
+              arguments:
+                values: ["run-up", "user-time", "machine-physics"]
+      - name: started_at
+        data_tests:
+          - not_null
+      - name: ended_at
+        data_tests:
+          - not_null

warehouses/accelerator/transform/accelerator/models/intermediate/int_cycles_start_end.sql

@@ -0,0 +1,14 @@
+with cycle_start_end as (
+
+  select
+
+    {{ adapter.quote('name') }},
+    min(started_at) as started_at,
+    max(ended_at) as ended_at
+
+  from {{ ref('int_cycle_phases_without_target') }}
+  group by {{ adapter.quote('name') }}
+
+)
+
+select * from cycle_start_end

warehouses/accelerator/transform/accelerator/models/intermediate/int_cycles_start_end.yml

@@ -0,0 +1,16 @@
+models:
+  - name: int_cycles_start_end
+    description: >
+      A single row per cycle defining the start and end dates of each cycle,
+      without any of the detailed phase information.
+    columns:
+      - name: name
+        data_tests:
+          - not_null
+          - unique
+      - name: started_at
+        data_tests:
+          - not_null
+      - name: ended_at
+        data_tests:
+          - not_null

warehouses/accelerator/transform/accelerator/models/operations/mcr_equipment_downtime_records.sql

@@ -6,116 +6,58 @@
     on_table_exists = 'drop'
 )
 }}
-{% set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") %}
 
 with
 
-staging_entries as ( select * from {{ ref('stg_opralogweb__user_entries') }} ),
-staging_chapter_entry as ( select * from {{ ref('stg_opralogweb__chapter_entry') }} ),
-staging_logbook_chapter as ( select * from {{ ref('stg_opralogweb__logbook_chapter') }} ),
-staging_logbooks as ( select * from {{ ref('stg_opralogweb__logbooks') }} ),
-staging_more_entry_columns as ( select * from {{ ref('stg_opralogweb__more_entry_columns') }} ),
-staging_additional_columns as ( select * from {{ ref('stg_opralogweb__additional_columns') }} ),
-staging_equipment_downtime_data_pre_opralog as (select * from {{ ref('stg_accelerator_sharepoint__equipment_downtime_data_pre_opralog') }} ),
+records_sharepoint as ( select * from {{ ref('stg_accelerator_sharepoint__mcr_equipment_downtime') }} ),
+records_opralogweb as ( select * from {{ ref('stg_opralogweb__mcr_equipment_downtime') }} ),
 
-denormalized as (
-  select
-    staging_entries.entry_id,
-    staging_entries.fault_occurred_at,
-    staging_entries.fault_date,
-    staging_additional_columns.column_title,
-    staging_more_entry_columns.string_data,
-    staging_more_entry_columns.number_data,
-    staging_entries.fault_description
-  from
-    staging_entries
-    join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id
-    join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no
-    join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id
-    left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id
-    left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id
-  where
-    staging_logbooks.logbook_name = {{ MCR_LOGBOOK }}
-    and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook
-    and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments')
-    and (
-      staging_more_entry_columns.string_data is not null
-      or staging_more_entry_columns.number_data is not null
-    )
-),
+records_sharepoint_with_cycle_phase_col as (
 
-downtime_records as (
   select
-    *
+
+    equipment,
+    fault_date,
+    cycle_name,
+    cast(NULL as varchar) as cycle_phase,
+    downtime_mins,
+    fault_occurred_at,
+    {{ adapter.quote('group') }},
+    fault_description,
+    managers_comments
+
   from
-    (
-      select
-
-        min(
-          case
-            column_title
-            when 'Equipment' then string_data
-          end
-        ) as equipment,
-        min(
-          case
-            column_title
-            when 'Lost Time' then number_data
-          end
-        ) as downtime_mins,
-        fault_date,
-        fault_occurred_at,
-        min(
-          case
-            column_title
-            when 'Group' then string_data
-          end
-        ) as {{ adapter.quote('group') }},
-        fault_description,
-        min(
-          case
-            column_title
-            when 'Group Leader comments' then string_data
-          end
-        ) as managers_comments
-      from
-        denormalized
-      group by
-        fault_occurred_at,
-        fault_date,
-        fault_description
-    )
-  where
-    equipment is not null
-    and downtime_mins is not null
-    and {{ adapter.quote('group') }} is not null
-
-  union all
-
-  select * from staging_equipment_downtime_data_pre_opralog
+
+    records_sharepoint
 
 ),
 
-downtime_records_with_cycle as (
+records_opralogweb_after_sharepoint_joined_with_cycles as (
+
   select
-    d.equipment,
-    d.fault_date,
+
+    r.equipment,
+    r.fault_date,
     c.name as cycle_name,
     c.phase as cycle_phase,
-    d.downtime_mins,
-    d.fault_occurred_at,
-    d.{{ adapter.quote('group') }},
-    d.fault_description,
-    d.managers_comments
+    r.downtime_mins,
+    r.fault_occurred_at,
+    r.{{ adapter.quote('group') }},
+    r.fault_description,
+    r.managers_comments
+
+  from records_opralogweb r
+  left join {{ ref("int_cycle_phases_without_target") }} c on r.fault_occurred_at between c.started_at and c.ended_at
+  where fault_occurred_at > (select max(fault_occurred_at) from records_sharepoint_with_cycle_phase_col)
+),
 
-  from
+unioned as (
+
+  select * from  records_sharepoint_with_cycle_phase_col
+  union
+  select * from records_opralogweb_after_sharepoint_joined_with_cycles
 
-    downtime_records d
-    left join {{ ref("cycles") }} c on d.fault_occurred_at between c.started_at and c.ended_at
 )
 
-select
-  *
-from
-  downtime_records_with_cycle
-order by fault_occurred_at asc
+-- add order by clause for iceberg table sorting crterion
+select * from unioned order by fault_occurred_at asc

warehouses/accelerator/transform/accelerator/models/operations/mcr_equipment_downtime_records.yml

@@ -0,0 +1,4 @@
+models:
+  - name: mcr_equipment_downtime_records
+    description: >
+      A full history of MCR equipment failures, including downtime and fault descriptions

warehouses/accelerator/transform/accelerator/models/operations/mean_time_between_failure.sql

@@ -0,0 +1,61 @@
+{{
+  config(
+    properties={
+      "partitioning": "ARRAY['cycle_name']",
+    },
+    on_table_exists = 'drop',
+)
+}}
+
+with
+
+cycles_start_end as ( select * from {{ ref('int_cycles_start_end') }} ),
+downtime_records as ( select * from {{ ref('mcr_equipment_downtime_records') }} ),
+
+equipment_up_at_col as (
+
+  select
+
+    cycle_name,
+    equipment,
+    fault_occurred_at,
+    downtime_mins,
+    fault_occurred_at + (interval '1' minute * downtime_mins) as equipment_up_at
+
+  from
+
+    downtime_records d
+
+  left join cycles_start_end c on d.cycle_name = c.name
+),
+
+uptime_col as (
+
+  select
+
+    *,
+    date_diff('minute',
+      lag(equipment_up_at, 1,
+          (select started_at from cycles_start_end where {{ adapter.quote('name') }} = cycle_name)
+         )
+         over
+         (partition by cycle_name, equipment order by fault_occurred_at), fault_occurred_at) as uptime_mins
+
+  from equipment_up_at_col
+),
+
+mean_time_between_failures as (
+
+  select
+
+    any_value(cycle_name) as cycle_name,
+    equipment,
+    sum(uptime_mins)/count(fault_occurred_at)/60. as mtbf_hours
+
+  from uptime_col
+  where cycle_name is not null
+  group by cycle_name, equipment
+
+)
+
+select * from mean_time_between_failures

warehouses/accelerator/transform/accelerator/models/operations/mean_time_between_failure.yml

@@ -0,0 +1,5 @@
+models:
+  - name: mean_time_between_failure
+    description: >
+      [Mean time between failures](https://en.wikipedia.org/wiki/Mean_time_between_failures)
+      by cycle and equipment.

warehouses/accelerator/transform/accelerator/models/operations/power_consumption.sql

@@ -11,7 +11,12 @@ with
 
 staged as (
 
-  select * from {{ ref('stg_electricity_sharepoint_rdm_data') }}
+  select
+
+    date_time as power_measured_at,
+    total_isis_power_mw
+
+  from {{ ref('stg_electricity_sharepoint_rdm_data') }}
 
 )
 

warehouses/accelerator/transform/accelerator/models/operations/power_consumption.yml

@@ -0,0 +1,11 @@
+models:
+  - name: power_consumption
+    description: >
+      The total power consumption for ISIS over time.
+    columns:
+      - name: power_measured_at
+        data_tests:
+          - not_null
+      - name: total_isis_power_mw
+        data_tests:
+          - not_null