Feature/dim-locations (#118)

* Add dim_locations. Issue #95

* Change stg_ports -> int_ports

* Start decoupling dim_ports and dim_connectors

* Add fact_location_capacity for location measures. Add location_key to fact_visits. Add port, connector and charge point counts to int_ports

* Build hardware dimension layer and rewire facts

- Add dim_charge_points, dim_ports, dim_connectors at correct grains
- Fix fan-out bugs in all three dims (were sourcing from connector-grain int_ports without dedup)
- Drop commissioned/decommissioned_ts from dim_ports and dim_connectors (charge point attributes, not port/connector)
- Rewire fact_downtime_daily, fact_uptime, fact_visits to reference dims instead of int_ports
- Add composite grain tests for dim_ports and dim_connectors
- Add semantic models for charge_points, ports, connectors; wire FK entities into charge_attempts, uptime, visits
- Update marts.yml docs to reflect all changes

* Revert int_ports to its grain and data.

Fix dim_charge_points adding port_count.

Add connector_count to dim_ports

* Add port_count section for dim_charge_points in marts.yml

* Fix connectors_count -> connector_count

* Change tests: to data_tests: project-wide according to new dbt naming

* Fix arguments syntax in marts.yml

* Change unit_tests mocking dims instead of int_ports

* MissingArgumentsPropertyInGenericTestDeprecation

 there's a deprecation warning (MissingArgumentsPropertyInGenericTestDeprecation, 2 occurrences) about dbt_utils.accepted_range on dim_ports needing its arguments nested under an arguments: property. Not an error, but worth fixing.

* demo to run full-refresh

The first run failed because the Docker volume had a stale fact_visits table with the old location_key column.

* demo ports test fix

* Revert "demo to run full-refresh"

This reverts commit e62adc4.

* Add decommissioned_ts and is_active to dim_ports

* Revert "Add decommissioned_ts and is_active to dim_ports"

This reverts commit 4527dba.

---------

Co-authored-by: Daria <daria@kwwhat.com>

Author: SYegor

demo/chat-bi/tests/decommissioned_ports_check.yml

@@ -2,5 +2,6 @@ name: decommissioned_ports_check
 prompt: "How many ports are currently decommissioned?"
 sql: |
   SELECT COUNT(*) AS decommissioned_ports
-  FROM analytics.ANALYTICS.dim_ports
-  WHERE decommissioned_ts IS NOT NULL
+  FROM analytics.ANALYTICS.dim_charge_points cp
+  JOIN analytics.ANALYTICS.dim_ports p ON cp.charge_point_id = p.charge_point_id
+  WHERE cp.decommissioned_ts IS NOT NULL

models/intermediate/int_ports.sql

@@ -5,12 +5,24 @@
   )
 }}
 
+with ports as (
+    select
+        charge_point_id,
+        location_id,
+        port_id,
+        connector_id,
+        connector_type,
+        commissioned_ts,
+        decommissioned_ts,
+    from {{ ref('stg_ports') }}
+)
+
 select
     charge_point_id,
     location_id,
     port_id,
     connector_id,
     connector_type,
     commissioned_ts,
-    decommissioned_ts
-from {{ ref('stg_ports') }}
+    decommissioned_ts,
+from ports

models/intermediate/intermediate.yml

@@ -6,32 +6,32 @@ models:
     columns:
       - name: charge_point_id
         description: "Identifier for the charge point"
-        tests:
+        data_tests:
           - not_null
       - name: connector_id
         description: "Identifier for the connector"
-        tests:
+        data_tests:
           - not_null
       - name: port_id
         description: "Identifier for the port"
-        tests:
+        data_tests:
           - not_null
       - name: latest_status
         description: "Most recent OCPP status for this connector"
-        tests:
+        data_tests:
           - not_null
           - accepted_values:
               arguments:
                 values: ['Available', 'Preparing', 'Charging', 'SuspendedEVSE', 'SuspendedEV', 'Finishing', 'Reserved', 'Unavailable', 'Faulted']
       - name: latest_error_code
         description: "Error code from the most recent StatusNotification"
-        tests:
+        data_tests:
           - not_null
       - name: latest_status_ts
         description: "Timestamp when the most recent status was ingested"
-        tests:
+        data_tests:
           - not_null
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:
@@ -44,30 +44,30 @@ models:
     columns:
       - name: charge_point_id
         description: "Identifier for the charge point"
-        tests:
+        data_tests:
           - not_null
       - name: unique_id
         description: "Unique identifier for the OCPP message"
-        tests:
+        data_tests:
           - not_null
       - name: connector_id
         description: "Identifier for the connector"
-        tests:
+        data_tests:
           - not_null
       - name: ingested_ts
         description: "Timestamp when the status notification was ingested"
-        tests:
+        data_tests:
           - not_null
       - name: status
         description: "Current status from the notification"
-        tests:
+        data_tests:
           - not_null
           - accepted_values:
               arguments:
                 values: ['Available', 'Preparing', 'Charging', 'SuspendedEVSE', 'SuspendedEV', 'Finishing', 'Reserved', 'Unavailable', 'Faulted']
       - name: error_code
         description: "Error code from the status notification payload (NoError if no error to report)"
-        tests:
+        data_tests:
           - not_null        
       - name: payload
         description: "Original payload from the status notification"
@@ -81,7 +81,7 @@ models:
         description: "Timestamp of the next status notification for the same charge point and connector (null for last status)"
       - name: confirmation_ingested_ts
         description: "Timestamp when the confirmation was ingested"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:
@@ -104,25 +104,25 @@ models:
     columns:
       - name: charge_point_id
         description: "Identifier for the charge point"
-        tests:
+        data_tests:
           - not_null
       - name: connector_id
         description: "Identifier for the charging connector"
-        tests:
+        data_tests:
           - not_null
       - name: unique_id
         description: "Unique identifier for the status change that triggered this charge attempt"
-        tests:
+        data_tests:
           - not_null
       - name: ingested_ts
         description: "Timestamp when the StatusNotification with status 'Preparing' was ingested (start of charge attempt)"
-        tests:
+        data_tests:
           - not_null
       - name: previous_status
         description: "Status before changing to 'Preparing'"
       - name: status
         description: "Current status (should be 'Preparing')"
-        tests:
+        data_tests:
           - not_null
           - accepted_values:
               arguments:
@@ -141,11 +141,11 @@ models:
         description: "Array of error codes from StatusNotification messages"
       - name: _unique_transaction_count
         description: "Count of unique transaction IDs for this charge attempt (should be 0 or 1)"
-        tests:
+        data_tests:
           - accepted_values:
               arguments:
                 values: [0, 1]
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:
@@ -158,23 +158,23 @@ models:
     columns:
       - name: transaction_id
         description: "Unique identifier for the transaction from OCPP StartTransaction/StopTransaction messages"
-        tests:
+        data_tests:
           - not_null
       - name: charge_point_id
         description: "Identifier for the charge point"
-        tests:
+        data_tests:
           - not_null
       - name: connector_id
         description: "Connector ID for this transaction"
-        tests:
+        data_tests:
           - not_null
       - name: ingested_ts
         description: "Earliest timestamp when any OCPP event for this transaction was ingested"
-        tests:
+        data_tests:
           - not_null
       - name: transaction_start_ts
         description: "Timestamp when the transaction started (from StartTransaction message)"
-        tests:
+        data_tests:
           - not_null
       - name: transaction_stop_ts
         description: "Timestamp when the transaction stopped (from StopTransaction message)"
@@ -193,11 +193,11 @@ models:
       - name: error_codes
         description: "Array of error codes from StatusNotification events that occurred during the transaction"
       - name: _unique_connectors_count
-        tests:
+        data_tests:
           - accepted_values:
               arguments:
                 values: [1]
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:
@@ -210,55 +210,55 @@ models:
     columns:
       - name: charge_point_id
         description: "Unique identifier for the charging point"
-        tests:
+        data_tests:
           - not_null
       - name: transaction_id
         description: "Unique identifier for the charging transaction"
-        tests:
+        data_tests:
           - not_null
       - name: connector_id
         description: "Unique identifier for the connector within the charging point"
-        tests:
+        data_tests:
           - not_null
       - name: ingested_ts
         description: "Timestamp when transaction was ingested into the system"
-        tests:
+        data_tests:
           - not_null
       - name: measurand
         description: "Type of measurement (e.g., Energy.Active.Import.Register, Voltage, Current.Import)"
-        tests:
+        data_tests:
           - not_null
       - name: unit
         description: "Unit of measurement (e.g., Wh, V, A, W)"
-        tests:
+        data_tests:
           - not_null
       - name: phase
         description: "Electrical phase (e.g., L1, L2, L3) or null for single-phase measurements"
       - name: first_measurement_ts
         description: "Meter timestamp of the first measurement for the transaction"
-        tests:
+        data_tests:
           - not_null
       - name: last_measurement_ts
         description: "Meter timestamp of the last measurement for the transaction"
-        tests:
+        data_tests:
           - not_null
       - name: min_value
         description: "Minimum value recorded for the transaction"
-        tests:
+        data_tests:
           - not_null
       - name: max_value
         description: "Maximum value recorded for the transaction"
-        tests:
+        data_tests:
           - not_null
       - name: avg_value
         description: "Average value for the transaction"
-        tests:
+        data_tests:
           - not_null
       - name: _count
         description: "Total number of measurements for the transaction"
-        tests:
+        data_tests:
           - not_null
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:
@@ -292,7 +292,7 @@ models:
         description: "Timestamp when the charge point was commissioned (null if not commissioned yet)"
       - name: decommissioned_ts
         description: "Timestamp when the charge point was decommissioned (null if still active)"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
             combination_of_columns:

models/marts/dim_charge_points.sql

@@ -0,0 +1,31 @@
+{{
+  config(
+    materialized='table',
+    description='Charge point dimension; one row per charge point.'
+  )
+}}
+
+with ports as (
+    select
+        charge_point_id,
+        location_id,
+        commissioned_ts,
+        decommissioned_ts,
+        count(distinct port_id) as port_count
+    from {{ ref('int_ports') }}
+    group by
+        charge_point_id,
+        location_id,
+        commissioned_ts,
+        decommissioned_ts
+)
+
+select
+    {{ dbt_utils.generate_surrogate_key(['ports.charge_point_id']) }} as charge_point_key,
+    ports.charge_point_id,
+    ports.location_id,
+    ports.commissioned_ts,
+    ports.decommissioned_ts,
+    ports.decommissioned_ts is null as is_active,
+    port_count,
+from ports

models/marts/dim_connectors.sql

@@ -0,0 +1,45 @@
+{{
+  config(
+    materialized='table',
+    description='Connector dimension; full refresh from int_ports'
+  )
+}}
+
+with ports as (
+    select
+        charge_point_id,
+        location_id,
+        port_id,
+        connector_id,
+        connector_type,
+    from {{ ref('int_ports') }}
+),
+
+latest_status as (
+    select
+        charge_point_id,
+        connector_id,
+        latest_status,
+        latest_error_code,
+        latest_status_ts,
+    from {{ ref('int_connector_latest_status') }}
+)
+
+select
+    {{ dbt_utils.generate_surrogate_key([
+        'ports.charge_point_id', 
+        'ports.port_id', 
+        'ports.connector_id'
+        ]) }} as connector_key,
+    ports.charge_point_id,
+    ports.location_id,
+    ports.port_id,
+    ports.connector_id,
+    ports.connector_type,
+    latest_status.latest_status,
+    latest_status.latest_error_code,
+    latest_status.latest_status_ts,
+from ports
+left join latest_status
+    on ports.charge_point_id = latest_status.charge_point_id
+    and ports.connector_id = latest_status.connector_id

models/marts/dim_locations.sql

@@ -0,0 +1,17 @@
+{{
+  config(
+    materialized='table',
+    description="Conformed location dimension. One row per distinct charging location. SCD Type 1."
+  )
+}}
+
+with locations as (
+    select distinct
+        location_id,
+    from {{ ref('int_ports') }}
+)
+
+select
+    {{ dbt_utils.generate_surrogate_key(['location_id']) }} as location_key,
+    location_id,
+from locations