feat: tinybird pipes and datasources for leaderboards (IN-813) (#3575)

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/activities.datasource

@@ -16,6 +16,7 @@ DESCRIPTION >
     - `url` is the direct link to the activity on the source platform (empty string if not available).
     - Sentiment analysis fields (`sentimentLabel`, `sentimentScore*`) provide sentiment metrics (-1 default for no analysis).
     - Git-specific fields (`git*`) track code changes, branch info, and merge status for code-related activities (0 default for non-git).
+
 TAGS "Activity preprocessing pipeline"
 
 SCHEMA >
@@ -46,8 +47,8 @@ SCHEMA >
     `createdById` UUID `json:$.createdById` DEFAULT toUUID('00000000-0000-0000-0000-000000000000'),
     `updatedById` UUID `json:$.updatedById` DEFAULT toUUID('00000000-0000-0000-0000-000000000000'),
     `segmentId` LowCardinality(String) `json:$.segmentId` DEFAULT ''
-    
-ENGINE "ReplacingMergeTree"
-ENGINE_PARTITION_KEY "toYear(createdAt)"
-ENGINE_SORTING_KEY "id"
-ENGINE_VER "updatedAt"
+
+ENGINE ReplacingMergeTree
+ENGINE_PARTITION_KEY toYear(createdAt)
+ENGINE_SORTING_KEY id
+ENGINE_VER updatedAt

services/libs/tinybird/datasources/activities_deduplicated_ds.datasource

@@ -1,5 +1,5 @@
 DESCRIPTION >
-     - `activities_deduplicated_ds` contains deduplicated raw activity events without relationship data.
+    - `activities_deduplicated_ds` contains deduplicated raw activity events without relationship data.
     - Created via copy pipe from `activities` datasource with deduplication and field selection for performance.
     - Since aggregations are mainly done on relationships, `activityRelations_deduplicated_cleaned_ds` should be used for reporting purposes instead.
     - Optimized subset of activity fields focused on core analytics needs.
@@ -9,11 +9,12 @@ DESCRIPTION >
     - `type` specifies the activity type (issues-opened, pull-request-opened, etc.) using LowCardinality.
     - `channel` contains the repository, channel, or forum where activity occurred.
     - `sourceId` is the unique identifier from the source platform.
-     - `score` is the computed importance/impact score for the activity.
+    - `score` is the computed importance/impact score for the activity.
     - `attributes` contains additional JSON metadata specific to the activity type.
     - `body` contains the activity’s textual body/content when applicable; empty string if not applicable.
     - `title` contains the activity’s title/subject when applicable; empty string if not applicable.
     - `url` is the direct link to the activity on the source platform; empty string if not available.
+
 TAGS "Activity preprocessing pipeline"
 
 SCHEMA >
@@ -31,11 +32,10 @@ SCHEMA >
     `url` String DEFAULT '',
     `updatedAt` DateTime64(3)
 
-ENGINE "MergeTree"
-ENGINE_PARTITION_KEY "toYear(timestamp)"
-ENGINE_SORTING_KEY "id, platform, channel"
-
-
 INDEXES >
-    idx_body_ngram3 body TYPE ngrambf_v1(3, 2048, 6, 0) GRANULARITY 64,
-    idx_title_ngram3 title TYPE ngrambf_v1(3, 512, 6, 0) GRANULARITY 64
+    idx_body_ngram3 body TYPE ngrambf_v1(3, 2048, 6, 0) GRANULARITY 64
+    idx_title_ngram3 title TYPE ngrambf_v1(3, 512, 6, 0) GRANULARITY 64
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY id, platform, channel

services/libs/tinybird/datasources/issues_analyzed.datasource

@@ -8,14 +8,17 @@ DESCRIPTION >
     - `closedAt` is the timestamp when the issue was closed (nullable for open issues).
     - `closedInSeconds` is the computed duration from open to close in seconds (nullable for open issues).
 
-TAGS "Issue analytics", "Performance metrics"
+TAGS "Performance metrics", "Issue analytics"
 
 SCHEMA >
     `id` String,
     `sourceId` String,
+    `segmentId` String,
     `openedAt` DateTime64(3),
+    `commentedAt` Nullable(DateTime64(3)),
     `closedAt` Nullable(DateTime64(3)),
-    `closedInSeconds` Nullable(Int64)
+    `closedInSeconds` Nullable(Int64),
+    `respondedInSeconds` Nullable(Int64)
 
 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYear(openedAt)

services/libs/tinybird/datasources/leaderboards_copy_ds.datasource

@@ -0,0 +1,30 @@
+DESCRIPTION >
+    - `leaderboards_copy_ds` contains pre-computed leaderboard rankings for open source projects across multiple categories.
+    - Created via copy pipe that runs daily at 1 AM, combining data from multiple leaderboard metric pipes.
+    - Supports six leaderboard types: longest-running, active-contributors, active-organizations, commit-activity, focused-teams, and small-teams-massive-output.
+    - Designed for high-performance read access to power project ranking and comparison features.
+    - `rank` is the position of the project within its leaderboard type (1 = top ranked).
+    - `id` is the unique project identifier.
+    - `segmentId` is the internal segment identifier for the project.
+    - `name` is the human-readable project name.
+    - `slug` is the URL-friendly project identifier used for filtering and routing.
+    - `logoUrl` is the URL to the project's logo image.
+    - `leaderboardType` identifies the ranking category (e.g., 'commit-activity', 'active-contributors').
+    - `value` is the primary metric value used for ranking (interpretation depends on leaderboardType).
+    - `previousPeriodValue` is the comparison metric from the previous period (0 for non-temporal leaderboards).
+
+TAGS "Analytics, Leaderboards, Project rankings"
+
+SCHEMA >
+    `rank` UInt64,
+    `id` String,
+    `segmentId` String,
+    `name` String,
+    `slug` String,
+    `logoUrl` String,
+    `leaderboardType` String,
+    `value` Float64,
+    `previousPeriodValue` Float64
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY leaderboardType, rank, slug, id

services/libs/tinybird/pipes/activities_backup_cleaned_mv.pipe

@@ -5,7 +5,7 @@ DESCRIPTION >
 
 NODE activities_backup_mv_v1
 SQL >
-    SELECT 
+    SELECT
         id,
         type,
         timestamp,

services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe

@@ -3,29 +3,44 @@ DESCRIPTION >
 
 NODE issues_opened
 SQL >
-    SELECT activityId as id, sourceId, timestamp AS openedAt
+    SELECT activityId as id, sourceId, timestamp AS openedAt, segmentId
     FROM activityRelations_deduplicated_cleaned_ds
     WHERE type = 'issues-opened'
 
 NODE issues_closed
 SQL >
     SELECT sourceParentId, MIN(timestamp) AS closedAt
     FROM activityRelations_deduplicated_cleaned_ds
-    WHERE type = 'issues-closed'
+    WHERE type = 'issues-closed' AND sourceParentId != ''
+    GROUP BY sourceParentId
+
+NODE issues_comment
+SQL >
+    SELECT sourceParentId, MIN(timestamp) AS commentedAt
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE type = 'issue-comment' AND sourceParentId != '' AND toYear(timestamp) >= 1971
     GROUP BY sourceParentId
 
 NODE issue_analysis_results_merged
 SQL >
     SELECT
         opened.id,
         opened.sourceId,
+        opened.segmentId as segmentId,
         opened.openedAt,
+        comment.commentedAt,
         IF(closed.closedAt = toDateTime(0), NULL, closed.closedAt) AS closedAt,
         IF(
             closedAt IS NULL, NULL, toUnixTimestamp(closed.closedAt) - toUnixTimestamp(opened.openedAt)
-        ) AS closedInSeconds
+        ) AS closedInSeconds,
+        IF(
+            commentedAt IS NULL,
+            NULL,
+            toUnixTimestamp(comment.commentedAt) - toUnixTimestamp(opened.openedAt)
+        ) AS respondedInSeconds
     FROM issues_opened opened
     LEFT JOIN issues_closed AS closed ON opened.sourceId = closed.sourceParentId
+    LEFT JOIN issues_comment AS comment ON opened.sourceId = comment.sourceParentId
 
 TYPE COPY
 TARGET_DATASOURCE issues_analyzed

services/libs/tinybird/pipes/leaderboards.pipe

@@ -0,0 +1,47 @@
+DESCRIPTION >
+    Main leaderboard query endpoint that retrieves leaderboard data from the consolidated datasource.
+    Supports filtering by project slug, leaderboard type, and search term. Includes pagination support.
+
+NODE leaderboards_list
+DESCRIPTION >
+    Queries the unified leaderboards datasource with optional filtering and pagination
+
+SQL >
+    %
+    SELECT *
+    FROM leaderboards_copy_ds
+    WHERE
+        1 = 1
+        {% if defined(slug) %}
+            AND slug = {{ String(slug, description="Project slug", required=False) }}
+        {% end %}
+        {% if defined(leaderboardType) %}
+            AND leaderboardType
+            = {{ String(leaderboardType, description="Leaderboard type", required=False) }}
+        {% end %}
+        {% if defined(search) %}
+            AND name
+            ilike '%'
+            || {{
+                String(
+                    search,
+                    description="Search project leaderboard open ended wildcard using slug",
+                    required=False,
+                )
+            }}
+            || '%'
+        {% end %}
+        {% if defined(maxRank) %}
+            AND rank
+            <= {{
+                Int32(
+                    maxRank,
+                    description="Define max rank to send",
+                    required=False,
+                )
+            }}
+        {% end %}
+    ORDER BY rank ASC
+    {% if not defined(maxRank) %}
+        LIMIT {{ Int32(pageSize, 20) }} OFFSET {{ Int32(page, 0) * Int32(pageSize, 20) }}
+    {% end %}

services/libs/tinybird/pipes/leaderboards_avg_commits_per_author.pipe

@@ -0,0 +1,48 @@
+DESCRIPTION >
+    Leaderboard ranking projects by average commits per author (focused teams). Calculates the ratio
+    of total commits to unique contributors, identifying projects with highly productive individual
+    contributors. Only includes projects with git platform integrations.
+
+NODE leaderboards_avg_commits_per_author_projects
+DESCRIPTION >
+    Retrieves all projects that have git integration enabled
+
+SQL >
+    SELECT id, name, slug, segmentId, logoUrl
+    FROM insights_projects_populated_ds
+    WHERE has(connectedPlatforms, 'git')
+    GROUP BY id, name, slug, segmentId, logoUrl
+
+NODE leaderboards_avg_commits_per_author_activities
+DESCRIPTION >
+    Counts total commits and unique authors per project for calculating average commits per author
+
+SQL >
+    SELECT segmentId, count() as commits, uniq(memberId) as unique_authors
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE
+        activityId != ''
+        AND (type = 'authored-commit' OR type = 'committed-commit')
+        AND platform = 'git'
+    GROUP BY segmentId
+    HAVING unique_authors > 0
+
+NODE leaderboards_avg_commits_per_author_results
+DESCRIPTION >
+    Calculates average commits per author ratio and ranks projects by highest productivity per contributor
+
+SQL >
+    SELECT
+        row_number() OVER (
+            ORDER BY
+                CAST(coalesce(c.commits, 0) AS Float64)
+                / CAST(coalesce(c.unique_authors, 1) AS Float64) DESC
+        ) as rank,
+        p.id,
+        p.segmentId,
+        p.name,
+        p.slug,
+        p.logoUrl,
+        (coalesce(c.commits, 0) / coalesce(c.unique_authors, 1)) as value
+    FROM leaderboards_avg_commits_per_author_projects p
+    INNER JOIN leaderboards_avg_commits_per_author_activities c ON p.segmentId = c.segmentId

services/libs/tinybird/pipes/leaderboards_codebase_size.pipe

@@ -0,0 +1,42 @@
+DESCRIPTION >
+    Leaderboard ranking projects by codebase size, measured as the net lines of code changed
+    (total insertions minus deletions) across all git commits. Only includes projects with
+    git platform integrations.
+
+NODE leaderboards_codebase_size_projects
+DESCRIPTION >
+    Retrieves all projects that have git integration enabled
+
+SQL >
+    SELECT id, name, slug, segmentId, logoUrl
+    FROM insights_projects_populated_ds
+    WHERE has(connectedPlatforms, 'git')
+    GROUP BY id, name, slug, segmentId, logoUrl
+
+NODE leaderboards_codebase_size_activities
+DESCRIPTION >
+    Calculates net lines changed per project by summing git insertions and subtracting deletions
+
+SQL >
+    SELECT segmentId, SUM(gitInsertions) - SUM(gitDeletions) AS lineDifference
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE platform = 'git' AND (gitInsertions > 0 OR gitDeletions > 0)
+    GROUP BY segmentId
+
+NODE leaderboards_codebase_size_result
+DESCRIPTION >
+    Joins project metadata with codebase size metrics and assigns rankings based on total lines changed
+
+SQL >
+    SELECT
+        row_number() OVER (ORDER BY coalesce(o.linesChanged, 0) DESC) as rank,
+        p.id,
+        p.segmentId,
+        p.name,
+        p.slug,
+        p.logoUrl,
+        CAST(coalesce(o.linesChanged, 0) as Float64) as value
+    FROM leaderboards_codebase_size_projects p
+    INNER JOIN leaderboards_codebase_size_activities o ON p.segmentId = o.segmentId
+    WHERE o.linesChanged > 0
+    ORDER BY value DESC

services/libs/tinybird/pipes/leaderboards_commits.pipe

@@ -0,0 +1,63 @@
+DESCRIPTION >
+    Leaderboard ranking projects by total commit activity. Compares the current period (last 12 months)
+    with the previous period (12-24 months ago). Higher commit counts rank higher, showing the most
+    active development projects.
+
+NODE leaderboards_commits_projects
+DESCRIPTION >
+    Retrieves all projects from the populated datasource
+
+SQL >
+    SELECT id, name, slug, segmentId, logoUrl
+    FROM insights_projects_populated_ds
+    GROUP BY id, name, slug, segmentId, logoUrl
+
+NODE leaderboards_commits_current_period
+DESCRIPTION >
+    Counts total git commits for each project in the last 12 months
+
+SQL >
+    SELECT segmentId, count(activityId) as commits
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE
+        timestamp >= now() - INTERVAL 12 MONTH
+        AND timestamp < now()
+        AND activityId != ''
+        AND (type = 'authored-commit' OR type = 'committed-commit')
+        AND platform = 'git'
+    GROUP BY segmentId
+
+NODE leaderboards_commits_previous_period
+DESCRIPTION >
+    Counts total git commits for each project in the previous 12 months (12-24 months ago)
+
+SQL >
+    SELECT segmentId, count(activityId) as commits
+    FROM activityRelations_deduplicated_cleaned_ds
+    WHERE
+        timestamp >= now() - INTERVAL 24 MONTH
+        AND timestamp < now() - INTERVAL 12 MONTH
+        AND activityId != ''
+        AND (type = 'authored-commit' OR type = 'committed-commit')
+        AND platform = 'git'
+    GROUP BY segmentId
+
+NODE leaderboards_copy_commits
+DESCRIPTION >
+    Joins project metadata with current and previous period commit counts, ranks by most active
+
+SQL >
+    SELECT
+        row_number() OVER (ORDER BY coalesce(o.commits, 0) DESC) as rank,
+        p.id,
+        p.segmentId,
+        p.name,
+        p.slug,
+        p.logoUrl,
+        cast(coalesce(o.commits, 0) as Float64) as value,
+        cast(coalesce(pp.commits, 0) as Float64) as previousPeriodValue
+    FROM leaderboards_commits_projects p
+    INNER JOIN leaderboards_commits_current_period o ON p.segmentId = o.segmentId
+    LEFT JOIN leaderboards_commits_previous_period pp ON p.segmentId = pp.segmentId
+    WHERE o.commits > 0
+    ORDER BY value DESC