refactor: activity heatmap pipe optimized by pre calculating local activity times (#3216)

Author: epipav

services/libs/tinybird/datasources/contributions_with_local_time_ds.datasource

@@ -0,0 +1,12 @@
+SCHEMA >
+    `id` String,
+    `timestamp` DateTime,
+    `channel` String,
+    `segmentId` String,
+    `timezone_offset` Int16,
+    `weekday` UInt8,
+    `two_hours_block` UInt16
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY segmentId, timestamp, weekday, two_hours_block

services/libs/tinybird/pipes/activity_heatmap_by_weekday_and_2hours_blocks.pipe

@@ -1,35 +1,24 @@
 DESCRIPTION >
     Serves the `Development - Contributions outside work hours` widget
 
-NODE country_mapping_array
-SQL >
-    SELECT groupArray((country, flag, country_code, timezone_offset)) AS country_data
-    FROM country_mapping
-
-NODE members_with_location_information
-SQL >
-    SELECT
-        m.id,
-        m.location,
-        arrayFilter(
-            x -> position(coalesce(nullIf(upper(m.country), ''), upper(m.location)), upper(x .1)) > 0,
-            (SELECT country_data FROM country_mapping_array)
-        ) AS matched_countries,
-        arrayJoin(
-            if(empty(matched_countries), [('Unknown', '❓', 'XX', 0)], matched_countries)
-        ) AS country_data
-    FROM members_sorted AS m
-    where country_data .1 != 'Unknown' and m.id in (select memberId from activities_filtered)
-
 NODE activities_with_local_timestamp
 SQL >
-    select
-        count(id) as activityCount,
-        toDayOfWeek(addHours(af.timestamp, mwli.country_data .4)) as weekday,
-        intDiv(toHour(addHours(af.timestamp, mwli.country_data .4)), 2) * 2 AS two_hours_block
-    from activities_filtered af
-    join members_with_location_information mwli on mwli.id = af.memberId
-    where platform in ('git', 'github', 'gitlab', 'gerrit')
+    %
+    select count(id) as activityCount, weekday, two_hours_block
+    from contributions_with_local_time_ds a
+    where
+        segmentId = (SELECT segmentId FROM segments_filtered)
+        {% if defined(startDate) %}
+            AND a.timestamp
+            > {{ DateTime(startDate, description="Filter activity timestamp after", required=False) }}
+        {% end %}
+        {% if defined(endDate) %}
+            AND a.timestamp
+            < {{ DateTime(endDate, description="Filter activity timestamp before", required=False) }}
+        {% end %}
+        {% if defined(repo) %}
+            AND a.channel = {{ String(repo, description="Filter activity repo", required=False) }}
+        {% end %}
     group by weekday, two_hours_block
 
 NODE weekday_hours

services/libs/tinybird/pipes/contributions_with_local_time.pipe

@@ -0,0 +1,35 @@
+NODE country_mapping_array
+SQL >
+    SELECT groupArray((country, country_code, timezone_offset)) AS country_data
+    FROM country_mapping_no_flags_ds
+
+NODE members_with_location_information
+SQL >
+    WITH
+        upperUTF8(coalesce(nullIf(m.country, ''), m.location)) AS search_u,
+        (SELECT arrayMap(x -> upperUTF8(x .1), country_data) FROM country_mapping_array) AS names_u,
+        (SELECT country_data FROM country_mapping_array) AS mapping,
+        multiSearchFirstIndexUTF8(search_u, names_u) AS idx,
+        if(idx = 0, ('Unknown', 'XX', 0), mapping[idx]) AS country_data
+    SELECT m.id, m.location, m.country, country_data
+    FROM members_sorted AS m
+    WHERE idx != 0
+
+NODE result
+SQL >
+    select
+        af.id,
+        af.timestamp,
+        af.channel,
+        af.segmentId,
+        mwli.country_data .3 as timezone_offset,
+        toDayOfWeek(addHours(af.timestamp, mwli.country_data .3)) as weekday,
+        intDiv(toHour(addHours(af.timestamp, mwli.country_data .3)), 2) * 2 AS two_hours_block
+    from activities_with_relations_sorted_deduplicated_ds af
+    join members_with_location_information mwli on mwli.id = af.memberId
+    where platform in ('git', 'github', 'gitlab', 'gerrit')
+
+TYPE COPY
+TARGET_DATASOURCE contributions_with_local_time_ds
+COPY_MODE replace
+COPY_SCHEDULE 55 * * * *