feat: allow filtering by collection slug on leaderboards (#3636)

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/leaderboards_copy_ds.datasource

@@ -12,6 +12,8 @@ DESCRIPTION >
     - `leaderboardType` identifies the ranking category (e.g., 'commit-activity', 'active-contributors').
     - `value` is the primary metric value used for ranking (interpretation depends on leaderboardType).
     - `previousPeriodValue` is the comparison metric from the previous period (0 for non-temporal leaderboards).
+    - `collectionsSlugs` list of collection slugs the project belongs to.
+    - `isLF` indicates whether the project is a LF project (1 = true, 0 = false).
 
 TAGS "Analytics, Leaderboards, Project rankings"
 
@@ -24,7 +26,9 @@ SCHEMA >
     `logoUrl` String,
     `leaderboardType` String,
     `value` Float64,
-    `previousPeriodValue` Float64
+    `previousPeriodValue` Float64,
+    `collectionsSlugs` Array(String),
+    `isLF` UInt8
 
 ENGINE MergeTree
 ENGINE_SORTING_KEY leaderboardType, rank, slug, id

services/libs/tinybird/pipes/leaderboards.pipe

@@ -19,6 +19,12 @@ SQL >
             AND leaderboardType
             = {{ String(leaderboardType, description="Leaderboard type", required=False) }}
         {% end %}
+        {% if defined(collectionSlug) %}
+            AND has(
+                collectionsSlugs,
+                {{ String(collectionSlug, description="Filter collection by slug", required=False) }}
+            )
+        {% end %}
         {% if defined(search) %}
             AND name
             ilike '%'

services/libs/tinybird/pipes/leaderboards_avg_commits_per_author.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have git integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
     WHERE has(connectedPlatforms, 'git')
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_avg_commits_per_author_activities
 DESCRIPTION >
@@ -44,6 +44,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         (coalesce(c.commits, 0) / coalesce(c.unique_authors, 1)) as value
     FROM leaderboards_avg_commits_per_author_projects p
     INNER JOIN leaderboards_avg_commits_per_author_activities c ON p.segmentId = c.segmentId

services/libs/tinybird/pipes/leaderboards_codebase_size.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have git integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
     WHERE has(connectedPlatforms, 'git')
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_codebase_size_activities
 DESCRIPTION >
@@ -38,6 +38,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         CAST(coalesce(o.lineDifference, 0) as Float64) as value
     FROM leaderboards_codebase_size_projects p
     INNER JOIN leaderboards_codebase_size_activities o ON p.segmentId = o.segmentId

services/libs/tinybird/pipes/leaderboards_commits.pipe

@@ -8,9 +8,9 @@ DESCRIPTION >
     Retrieves all projects from the populated datasource
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_commits_current_period
 DESCRIPTION >
@@ -56,6 +56,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         cast(coalesce(o.commits, 0) as Float64) as value,
         cast(coalesce(pp.commits, 0) as Float64) as previousPeriodValue
     FROM leaderboards_commits_projects p

services/libs/tinybird/pipes/leaderboards_issue_response.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have GitHub or GitLab integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
     WHERE arrayExists(x -> x IN ('github', 'gitlab'), connectedPlatforms)
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_issue_response_current_period
 DESCRIPTION >
@@ -55,6 +55,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         coalesce(o.averageResponseTimeSeconds, 0) as value,
         coalesce(pp.averageResponseTimeSeconds, 0) as previousPeriodValue
     FROM leaderboards_issue_response_projects p

services/libs/tinybird/pipes/leaderboards_members.pipe

@@ -19,10 +19,10 @@ SQL >
 
 NODE leaderboards_members_current_period
 DESCRIPTION >
-    Counts total contribution activities per member for the last 12 months
+    Counts total contribution activities per member for the last 12 months and collects segment IDs
 
 SQL >
-    SELECT memberId, count(*) AS memberActivityCount
+    SELECT memberId, count(*) AS memberActivityCount, groupUniqArray(segmentId) as segmentIds
     FROM activityRelations_deduplicated_cleaned_ds ar
     INNER JOIN
         leaderboards_member_activity_types at ON ar.type = at.activityType AND ar.platform = at.platform
@@ -49,9 +49,16 @@ SQL >
         AND timestamp < now() - INTERVAL 12 MONTH
     GROUP BY memberId
 
+NODE leaderboards_members_projects
+DESCRIPTION >
+    Get project details for collection slugs lookup
+
+SQL >
+    SELECT segmentId, collectionsSlugs FROM insights_projects_populated_ds WHERE segmentId != ''
+
 NODE leaderboards_members_results
 DESCRIPTION >
-    Joins member metadata with current and previous period activity counts, ranks by most activities
+    Joins member metadata with current and previous period activity counts, ranks by most activities, and aggregates collection slugs
 
 SQL >
     SELECT
@@ -61,11 +68,16 @@ SQL >
         p.displayName as name,
         '' as slug,
         p.avatar as logoUrl,
+        arrayDistinct(arrayFlatten(groupArray(proj.collectionsSlugs))) as collectionSlugs,
+        0 as isLF,
         cast(coalesce(c.memberActivityCount, 0) as Float64) as value,
         cast(coalesce(pp.memberActivityCount, 0) as Float64) as previousPeriodValue
     FROM leaderboards_members_data p
     INNER JOIN leaderboards_members_current_period c ON p.id = c.memberId
-    LEFT JOIN leaderboards_members_previous_period pp ON p.id = pp.memberId
+    LEFT JOIN leaderboards_members_previous_period pp ON p.id = pp.memberId ARRAY
+    JOIN c.segmentIds as sid
+    LEFT JOIN leaderboards_members_projects proj ON proj.segmentId = sid
     WHERE c.memberActivityCount > 0
+    GROUP BY p.id, p.displayName, p.avatar, c.memberActivityCount, pp.memberActivityCount
     ORDER BY value DESC
     LIMIT 100

services/libs/tinybird/pipes/leaderboards_merge_time.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have GitHub or GitLab integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
     WHERE arrayExists(x -> x IN ('github', 'gitlab'), connectedPlatforms)
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_merge_time_current_period
 DESCRIPTION >
@@ -50,6 +50,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         cast(coalesce(o.averageMergeTimeSeconds, 0) as Float64) as value,
         cast(coalesce(pp.averageMergeTimeSeconds, 0) as Float64) as previousPeriodValue
     FROM leaderboards_merge_time_projects p

services/libs/tinybird/pipes/leaderboards_organizations.pipe

@@ -19,10 +19,11 @@ SQL >
 
 NODE leaderboards_organizations_current_period
 DESCRIPTION >
-    Counts total contribution activities per organization for the last 12 months
+    Counts total contribution activities per organization for the last 12 months and collects segment IDs
 
 SQL >
-    SELECT organizationId, count(*) AS organizationActivityCount
+    SELECT
+        organizationId, count(*) AS organizationActivityCount, groupUniqArray(segmentId) as segmentIds
     FROM activityRelations_deduplicated_cleaned_ds ar
     INNER JOIN
         leaderboards_organizations_activity_types at
@@ -53,9 +54,16 @@ SQL >
         AND timestamp < now() - INTERVAL 12 MONTH
     GROUP BY organizationId
 
+NODE leaderboards_organizations_projects
+DESCRIPTION >
+    Get project details for collection slugs lookup
+
+SQL >
+    SELECT segmentId, collectionsSlugs FROM insights_projects_populated_ds WHERE segmentId != ''
+
 NODE leaderboards_organizations_results
 DESCRIPTION >
-    Joins organization metadata with current and previous period activity counts, ranks by most activities
+    Joins organization metadata with current and previous period activity counts, ranks by most activities, and aggregates collection slugs
 
 SQL >
     SELECT
@@ -65,11 +73,16 @@ SQL >
         p.displayName as name,
         '' as slug,
         p.logo as logoUrl,
+        arrayDistinct(arrayFlatten(groupArray(proj.collectionsSlugs))) as collectionSlugs,
+        0 as isLF,
         cast(coalesce(c.organizationActivityCount, 0) as Float64) as value,
         cast(coalesce(pp.organizationActivityCount, 0) as Float64) as previousPeriodValue
     FROM leaderboards_organizations_data p
     INNER JOIN leaderboards_organizations_current_period c ON p.id = c.organizationId
-    LEFT JOIN leaderboards_organizations_previous_period pp ON p.id = pp.organizationId
+    LEFT JOIN leaderboards_organizations_previous_period pp ON p.id = pp.organizationId ARRAY
+    JOIN c.segmentIds as sid
+    LEFT JOIN leaderboards_organizations_projects proj ON proj.segmentId = sid
     WHERE c.organizationActivityCount > 0
+    GROUP BY p.id, p.displayName, p.logo, c.organizationActivityCount, pp.organizationActivityCount
     ORDER BY value DESC
     LIMIT 100

services/libs/tinybird/pipes/leaderboards_project_active_contributors.pipe

@@ -8,9 +8,9 @@ DESCRIPTION >
     Retrieves all projects from the populated datasource
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
     FROM insights_projects_populated_ds
-    GROUP BY id, name, slug, segmentId, logoUrl
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
 
 NODE leaderboards_project_active_contributors_activity_types
 DESCRIPTION >
@@ -65,6 +65,8 @@ SQL >
         p.name as name,
         p.slug as slug,
         p.logoUrl as logoUrl,
+        p.collectionsSlugs as collectionsSlugs,
+        p.isLF as isLF,
         cast(coalesce(c.contributor_count, 0) as Float64) as value,
         cast(coalesce(pp.contributor_count, 0) as Float64) as previousPeriodValue
     FROM leaderboards_project_active_contributors_projects p