feat: added the status field to all project related datasource and pipes (#3746)

Signed-off-by: Efren Lim <elim@linuxfoundation.org>

Author: emlimlf

services/libs/tinybird/datasources/insights_projects_populated_ds.datasource

@@ -21,6 +21,7 @@ DESCRIPTION >
     - `softwareValue` is the estimated economic value of the software (UInt64).
     - `contributorCount` and `organizationCount` are computed contributor and organization statistics (UInt64).
     - `healthScore` is the overall project health score (Float64).
+    - `status` is the status of the project ex: active or archived
 
 TAGS "Project metadata", "Analytics enrichment"
 
@@ -57,7 +58,8 @@ SCHEMA >
     `healthScore` Float64,
     `communityPlatforms` Array(String),
     `communityKeywords` Array(String),
-    `communityLanguages` Array(String)
+    `communityLanguages` Array(String),
+    `status` String
 
 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYear(createdAt)

services/libs/tinybird/datasources/leaderboards_copy_ds.datasource

@@ -30,7 +30,8 @@ SCHEMA >
     `previousPeriodValue` Float64,
     `collectionsSlugs` Array(String),
     `isLF` UInt8,
-    `githubHandle` String
+    `githubHandle` String,
+    `status` String
 
 ENGINE MergeTree
 ENGINE_SORTING_KEY leaderboardType, rank, slug, id

services/libs/tinybird/pipes/collections_list.pipe

@@ -104,7 +104,9 @@ SQL >
                         'slug',
                         toString(collections_featured_projects.slug),
                         'logo',
-                        toString(collections_featured_projects.logo)
+                        toString(collections_featured_projects.logo),
+                        'status',
+                        toString(collections_featured_projects.status)
                     )
                 )
             ) AS featuredProjects

services/libs/tinybird/pipes/insightsProjects_filtered.pipe

@@ -31,7 +31,8 @@ SQL >
         insights_projects_populated_ds.healthScore,
         insights_projects_populated_ds.communityPlatforms,
         insights_projects_populated_ds.communityKeywords,
-        insights_projects_populated_ds.communityLanguages
+        insights_projects_populated_ds.communityLanguages,
+        insights_projects_populated_ds.status
     FROM insights_projects_populated_ds
     where
         insights_projects_populated_ds.enabled = 1
@@ -95,4 +96,5 @@ SQL >
         insights_projects_populated_ds.healthScore,
         insights_projects_populated_ds.communityPlatforms,
         insights_projects_populated_ds.communityKeywords,
-        insights_projects_populated_ds.communityLanguages
+        insights_projects_populated_ds.communityLanguages,
+        insights_projects_populated_ds.status

services/libs/tinybird/pipes/insights_projects_populated_copy.pipe

@@ -169,7 +169,8 @@ SQL >
         archived_excluded_repositories.excludedRepositories as excludedRepositories,
         insights_projects_populated_copy_mentions.communityPlatforms as communityPlatforms,
         insights_projects_populated_copy_mentions.communityKeywords as communityKeywords,
-        insights_projects_populated_copy_mentions.communityLanguages as communityLanguages
+        insights_projects_populated_copy_mentions.communityLanguages as communityLanguages,
+        segments.status as status
     FROM insightsProjects FINAL
     LEFT JOIN
         insights_projects_populated_copy_collections_slugs
@@ -195,6 +196,7 @@ SQL >
     LEFT JOIN
         insights_projects_populated_copy_mentions
         ON insights_projects_populated_copy_mentions.projectSlug = insightsProjects.slug
+    LEFT JOIN segments ON segments.id = insightsProjects.segmentId
     WHERE isNull (insightsProjects.deletedAt)
 
 TYPE COPY

services/libs/tinybird/pipes/leaderboards_avg_commits_per_author.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have git integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
     FROM insights_projects_populated_ds
     WHERE has(connectedPlatforms, 'git')
-    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
 
 NODE leaderboards_avg_commits_per_author_activities
 DESCRIPTION >
@@ -45,6 +45,7 @@ SQL >
         p.logoUrl as logoUrl,
         p.collectionsSlugs as collectionsSlugs,
         p.isLF as isLF,
-        (coalesce(c.commits, 0) / coalesce(c.unique_authors, 1)) as value
+        (coalesce(c.commits, 0) / coalesce(c.unique_authors, 1)) as value,
+        p.status as status
     FROM leaderboards_avg_commits_per_author_projects p
     INNER JOIN leaderboards_avg_commits_per_author_activities c ON p.segmentId = c.segmentId

services/libs/tinybird/pipes/leaderboards_codebase_size.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have git integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
     FROM insights_projects_populated_ds
     WHERE has(connectedPlatforms, 'git')
-    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
 
 NODE leaderboards_codebase_size_activities
 DESCRIPTION >
@@ -37,7 +37,8 @@ SQL >
         p.logoUrl as logoUrl,
         p.collectionsSlugs as collectionsSlugs,
         p.isLF as isLF,
-        CAST(coalesce(o.lineDifference, 0) as Float64) as value
+        CAST(coalesce(o.lineDifference, 0) as Float64) as value,
+        p.status as status
     FROM leaderboards_codebase_size_projects p
     INNER JOIN leaderboards_codebase_size_activities o ON p.segmentId = o.segmentId
     WHERE o.lineDifference > 0

services/libs/tinybird/pipes/leaderboards_commits.pipe

@@ -8,9 +8,9 @@ DESCRIPTION >
     Retrieves all projects from the populated datasource
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
     FROM insights_projects_populated_ds
-    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
 
 NODE leaderboards_commits_current_period
 DESCRIPTION >
@@ -57,7 +57,8 @@ SQL >
         p.collectionsSlugs as collectionsSlugs,
         p.isLF as isLF,
         cast(coalesce(o.commits, 0) as Float64) as value,
-        cast(coalesce(pp.commits, 0) as Float64) as previousPeriodValue
+        cast(coalesce(pp.commits, 0) as Float64) as previousPeriodValue,
+        p.status as status
     FROM leaderboards_commits_projects p
     INNER JOIN leaderboards_commits_current_period o ON p.segmentId = o.segmentId
     LEFT JOIN leaderboards_commits_previous_period pp ON p.segmentId = pp.segmentId

services/libs/tinybird/pipes/leaderboards_forks.pipe

@@ -8,9 +8,9 @@ DESCRIPTION >
     Retrieves all projects from the populated datasource
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
     FROM insights_projects_populated_ds
-    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
 
 NODE leaderboards_forks_metrics
 DESCRIPTION >
@@ -36,7 +36,8 @@ SQL >
         p.collectionsSlugs as collectionsSlugs,
         p.isLF as isLF,
         cast(coalesce(m.currentForks, 0) as Float64) as value,
-        cast(coalesce(m.previousForks, 0) as Float64) as previousPeriodValue
+        cast(coalesce(m.previousForks, 0) as Float64) as previousPeriodValue,
+        p.status as status
     FROM leaderboards_forks_projects p
     INNER JOIN leaderboards_forks_metrics m ON p.id = m.id
     WHERE m.currentForks > 0

services/libs/tinybird/pipes/leaderboards_issue_response.pipe

@@ -8,10 +8,10 @@ DESCRIPTION >
     Retrieves all projects that have GitHub or GitLab integration enabled
 
 SQL >
-    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    SELECT id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
     FROM insights_projects_populated_ds
     WHERE arrayExists(x -> x IN ('github', 'gitlab'), connectedPlatforms)
-    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF
+    GROUP BY id, name, slug, segmentId, logoUrl, collectionsSlugs, isLF, status
 
 NODE leaderboards_issue_response_current_period
 DESCRIPTION >
@@ -58,7 +58,8 @@ SQL >
         p.collectionsSlugs as collectionsSlugs,
         p.isLF as isLF,
         coalesce(o.averageResponseTimeSeconds, 0) as value,
-        coalesce(pp.averageResponseTimeSeconds, 0) as previousPeriodValue
+        coalesce(pp.averageResponseTimeSeconds, 0) as previousPeriodValue,
+        p.status as status
     FROM leaderboards_issue_response_projects p
     INNER JOIN leaderboards_issue_response_current_period o ON p.segmentId = o.segmentId
     LEFT JOIN leaderboards_issue_response_previous_period pp ON p.segmentId = pp.segmentId