feat: add new columns to pr analyzed (IN-872) (#3662)

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/pull_request_analyzed_MV_ds.datasource

@@ -0,0 +1,30 @@
+SCHEMA >
+    `id` String,
+    `sourceId` String,
+    `openedAt` DateTime64(3),
+    `segmentId` String,
+    `channel` String,
+    `memberId` String,
+    `organizationId` String,
+    `gitChangedLinesBucket` String,
+    `assignedAt` Nullable(DateTime64(3)),
+    `reviewRequestedAt` Nullable(DateTime64(3)),
+    `reviewedAt` Nullable(DateTime64(3)),
+    `approvedAt` Nullable(DateTime64(3)),
+    `closedAt` Nullable(DateTime64(3)),
+    `mergedAt` Nullable(DateTime64(3)),
+    `resolvedAt` Nullable(DateTime64(3)),
+    `assignedInSeconds` Nullable(Int64),
+    `reviewRequestedInSeconds` Nullable(Int64),
+    `reviewedInSeconds` Nullable(Int64),
+    `closedInSeconds` Nullable(Int64),
+    `mergedInSeconds` Nullable(Int64),
+    `resolvedInSeconds` Nullable(Int64),
+    `platform` String,
+    `numberOfPatchsets` Nullable(Int64),
+    `snapshotId` DateTime
+
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYYYYMM(snapshotId)
+ENGINE_SORTING_KEY snapshotId, segmentId, channel, openedAt, gitChangedLinesBucket, sourceId, id
+ENGINE_TTL toDateTime(snapshotId) + toIntervalDay(1)

services/libs/tinybird/datasources/pull_requests_analyzed.datasource

@@ -10,6 +10,8 @@ DESCRIPTION >
     - `gitChangedLinesBucket` categorizes the PR size by lines changed.
     - `assignedAt`, `reviewRequestedAt`, `reviewedAt`, `approvedAt`, `closedAt`, `mergedAt`, `resolvedAt` track workflow stage timestamps (nullable).
     - `assignedInSeconds`, `reviewRequestedInSeconds`, `reviewedInSeconds`, `closedInSeconds`, `mergedInSeconds`, `resolvedInSeconds` are computed durations for each stage (nullable).
+    - `platform` is the source platform (GitHub, GitLab, Gerrit, etc.) inherited from activities.
+    - `numberOfPatchsets` is the count of patchsets for Gerrit changesets (nullable, only applicable to Gerrit).
 
 TAGS "Pull request analytics", "Developer workflow metrics"
 
@@ -35,6 +37,8 @@ SCHEMA >
     `closedInSeconds` Nullable(Int64),
     `mergedInSeconds` Nullable(Int64),
     `resolvedInSeconds` Nullable(Int64),
+    `platform` String,
+    `numberOfPatchsets` Nullable(Int64),
     `snapshotId` DateTime
 
 ENGINE MergeTree

services/libs/tinybird/pipes/pull_request_analysis_baseline_merge_MV.pipe

@@ -14,6 +14,7 @@ SQL >
         gitChangedLinesBucket,
         memberId,
         organizationId,
+        platform,
         updatedAt
     FROM activityRelations_enrich_clean_snapshot_MV_ds
     where
@@ -30,6 +31,7 @@ SQL >
             'merge_request-review-changes-requested',
             'changeset_comment-created',
             'patchset_comment-created',
+            'patchset-created',
             'pull_request-reviewed',
             'merge_request-review-approved',
             'patchset_approval-created',
@@ -84,6 +86,11 @@ SQL >
             timestamp,
             type IN ('pull_request-opened', 'merge_request-opened', 'changeset-created')
         ) AS organizationId,
+        argMinIf(
+            platform,
+            timestamp,
+            type IN ('pull_request-opened', 'merge_request-opened', 'changeset-created')
+        ) AS platform,
         minIf(
             timestamp, type IN ('pull_request-opened', 'merge_request-opened', 'changeset-created')
         ) AS openedAt,
@@ -92,6 +99,7 @@ SQL >
             timestamp,
             type IN ('pull_request-opened', 'merge_request-opened', 'changeset-created')
         ) AS openedUpdatedAt,
+        toInt64(countIf(type = 'patchset-created')) AS numberOfPatchsets,
         argMin(
             updatedAt, if(type IN ('pull_request-assigned', 'merge_request-assigned'), timestamp, NULL)
         ) AS assignedUpdatedAt,
@@ -243,6 +251,7 @@ SQL >
             'merge_request-review-changes-requested',
             'changeset_comment-created',
             'patchset_comment-created',
+            'patchset-created',
             'merge_request-review-approved',
             'patchset_approval-created',
             'pull_request-closed',
@@ -354,6 +363,12 @@ SQL >
         IF(
             resolvedAt IS NULL, NULL, toUnixTimestamp(resolvedAt) - toUnixTimestamp(openedAt)
         ) AS resolvedInSeconds,
+        if(existing.platform != '', existing.platform, new.platform) as platform,
+        if(
+            new.numberOfPatchsets > 0,
+            COALESCE(existing.numberOfPatchsets, 0) + new.numberOfPatchsets,
+            existing.numberOfPatchsets
+        ) as numberOfPatchsets,
         toStartOfInterval(
             greatest(
                 COALESCE(new.openedUpdatedAt, existing.snapshotId, toDateTime(0)),

services/libs/tinybird/pipes/pull_request_analysis_copy_pipe.pipe

@@ -11,7 +11,8 @@ SQL >
         segmentId,
         gitChangedLinesBucket,
         memberId,
-        organizationId
+        organizationId,
+        platform
     FROM activityRelations_deduplicated_cleaned_ds
     WHERE type = 'pull_request-opened' OR type = 'merge_request-opened' OR type = 'changeset-created'
 
@@ -86,6 +87,16 @@ SQL >
         OR type = 'changeset-abandoned'
     GROUP BY sourceParentId
 
+NODE patchsets_count
+DESCRIPTION >
+    Count the number of patchsets for each Gerrit changeset
+
+SQL >
+    SELECT sourceParentId, toInt64(COUNT(*)) AS numberOfPatchsets
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE type = 'patchset-created'
+    GROUP BY sourceParentId
+
 NODE pull_request_analysis_results_merged
 SQL >
     SELECT
@@ -125,7 +136,9 @@ SQL >
         ) AS mergedInSeconds,
         IF(
             resolvedAt IS NULL, NULL, toUnixTimestamp(resolvedAt) - toUnixTimestamp(openedAt)
-        ) AS resolvedInSeconds
+        ) AS resolvedInSeconds,
+        pr_opened.platform,
+        patchsets.numberOfPatchsets
     FROM pull_request_opened pr_opened
     LEFT JOIN pull_request_first_assigned AS assigned ON pr_opened.sourceId = assigned.sourceParentId
     LEFT JOIN
@@ -137,6 +150,7 @@ SQL >
     LEFT JOIN pull_request_first_closed AS closed ON pr_opened.sourceId = closed.sourceParentId
     LEFT JOIN pull_request_first_merged AS merged ON pr_opened.sourceId = merged.sourceParentId
     LEFT JOIN pull_request_first_resolved as resolved on pr_opened.sourceId = resolved.sourceParentId
+    LEFT JOIN patchsets_count AS patchsets ON pr_opened.sourceId = patchsets.sourceParentId
 
 TYPE COPY
 TARGET_DATASOURCE pull_requests_analyzed

services/libs/tinybird/pipes/pull_request_analysis_initial_snapshot.pipe

@@ -12,6 +12,7 @@ SQL >
         gitChangedLinesBucket,
         memberId,
         organizationId,
+        platform,
         updatedAt
     FROM activityRelations_deduplicated_cleaned_ds
     WHERE
@@ -111,6 +112,18 @@ SQL >
         )
     GROUP BY sourceParentId
 
+NODE patchsets_count
+DESCRIPTION >
+    Count the number of patchsets for each Gerrit changeset
+
+SQL >
+    SELECT sourceParentId, toInt64(COUNT(*)) AS numberOfPatchsets
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE
+        snapshotId = (select max(snapshotId) from activityRelations_deduplicated_cleaned_ds)
+        AND type = 'patchset-created'
+    GROUP BY sourceParentId
+
 NODE pull_request_analysis_results_merged
 SQL >
     SELECT
@@ -151,6 +164,8 @@ SQL >
         IF(
             resolvedAt IS NULL, NULL, toUnixTimestamp(resolvedAt) - toUnixTimestamp(openedAt)
         ) AS resolvedInSeconds,
+        pr_opened.platform,
+        patchsets.numberOfPatchsets,
         toStartOfInterval(
             greatest(
                 pr_opened.updatedAt,
@@ -176,6 +191,7 @@ SQL >
     LEFT JOIN pull_request_first_closed AS closed ON pr_opened.sourceId = closed.sourceParentId
     LEFT JOIN pull_request_first_merged AS merged ON pr_opened.sourceId = merged.sourceParentId
     LEFT JOIN pull_request_first_resolved as resolved on pr_opened.sourceId = resolved.sourceParentId
+    LEFT JOIN patchsets_count AS patchsets ON pr_opened.sourceId = patchsets.sourceParentId
 
 TYPE COPY
 TARGET_DATASOURCE pull_requests_analyzed