refactor: move last vulnerability scan status to projects list (#4022)

Signed-off-by: anilb <epipav@gmail.com>

Author: epipav

services/libs/tinybird/datasources/insights_projects_populated_ds.datasource

@@ -23,6 +23,7 @@ DESCRIPTION >
     - `contributorCount` and `organizationCount` are computed contributor and organization statistics (UInt64).
     - `healthScore` is the overall project health score (Float64).
     - `status` is the status of the project ex: active or archived
+    - `lastVulnerabilityScanStatus` is the aggregated vulnerability scan status across all project repositories (`LowCardinality(Nullable(String))`). Possible values: `'success'` (all scans passed), `'no_packages_found'` (no dependency packages detected), `'failure'` (at least one scan failed), `'running'` (scan in progress). NULL when no scan records exist for the project's repositories.
 
 TAGS "Project metadata", "Analytics enrichment"
 
@@ -61,7 +62,8 @@ SCHEMA >
     `communityPlatforms` Array(String),
     `communityKeywords` Array(String),
     `communityLanguages` Array(String),
-    `status` String
+    `status` String,
+    `lastVulnerabilityScanStatus` LowCardinality(Nullable(String))
 
 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYear(createdAt)

services/libs/tinybird/pipes/insightsProjects_filtered.pipe

@@ -33,7 +33,8 @@ SQL >
         insights_projects_populated_ds.communityPlatforms,
         insights_projects_populated_ds.communityKeywords,
         insights_projects_populated_ds.communityLanguages,
-        insights_projects_populated_ds.status
+        insights_projects_populated_ds.status,
+        insights_projects_populated_ds.lastVulnerabilityScanStatus
     FROM insights_projects_populated_ds
     where
         insights_projects_populated_ds.enabled = 1
@@ -99,4 +100,5 @@ SQL >
         insights_projects_populated_ds.communityPlatforms,
         insights_projects_populated_ds.communityKeywords,
         insights_projects_populated_ds.communityLanguages,
-        insights_projects_populated_ds.status
+        insights_projects_populated_ds.status,
+        insights_projects_populated_ds.lastVulnerabilityScanStatus

services/libs/tinybird/pipes/insights_projects_populated_copy.pipe

@@ -136,6 +136,58 @@ SQL >
     FROM mentions FINAL
     GROUP BY projectSlug
 
+NODE last_vulnerability_scan_status
+SQL >
+    SELECT
+        insights_projects_populated_copy_flatten_projects.id as insightsProjectId,
+        multiIf(
+            max(
+                multiIf(
+                    vs.status = 'success',
+                    4,
+                    vs.status = 'no_packages_found',
+                    3,
+                    vs.status = 'failure',
+                    2,
+                    1
+                )
+            )
+            = 4,
+            'success',
+            max(
+                multiIf(
+                    vs.status = 'success',
+                    4,
+                    vs.status = 'no_packages_found',
+                    3,
+                    vs.status = 'failure',
+                    2,
+                    1
+                )
+            )
+            = 3,
+            'no_packages_found',
+            max(
+                multiIf(
+                    vs.status = 'success',
+                    4,
+                    vs.status = 'no_packages_found',
+                    3,
+                    vs.status = 'failure',
+                    2,
+                    1
+                )
+            )
+            = 2,
+            'failure',
+            'running'
+        ) as lastVulnerabilityScanStatus
+    FROM insights_projects_populated_copy_flatten_projects
+    LEFT JOIN
+        vulnerability_scans vs final
+        ON vs.repoUrl = insights_projects_populated_copy_flatten_projects.repository
+    group by insightsProjectId
+
 NODE insights_projects_populated_copy_results
 DESCRIPTION >
     Join everything together
@@ -176,7 +228,8 @@ SQL >
         insights_projects_populated_copy_mentions.communityPlatforms as communityPlatforms,
         insights_projects_populated_copy_mentions.communityKeywords as communityKeywords,
         insights_projects_populated_copy_mentions.communityLanguages as communityLanguages,
-        segments.status as status
+        segments.status as status,
+        last_vulnerability_scan_status.lastVulnerabilityScanStatus as lastVulnerabilityScanStatus
     FROM insightsProjects FINAL
     LEFT JOIN
         insights_projects_populated_copy_collections_slugs
@@ -202,6 +255,9 @@ SQL >
     LEFT JOIN
         insights_projects_populated_copy_mentions
         ON insights_projects_populated_copy_mentions.projectSlug = insightsProjects.slug
+    LEFT JOIN
+        last_vulnerability_scan_status
+        ON last_vulnerability_scan_status.insightsProjectId = insightsProjects.id
     LEFT JOIN segments ON segments.id = insightsProjects.segmentId
     WHERE isNull (insightsProjects.deletedAt)
 

services/libs/tinybird/pipes/vulnerabilities_summary.pipe

@@ -1,54 +1,3 @@
-NODE last_scan_status
-SQL >
-    %
-    SELECT
-        multiIf(
-            max(
-                multiIf(
-                    s.status = 'success',
-                    4,
-                    s.status = 'no_packages_found',
-                    3,
-                    s.status = 'failure',
-                    2,
-                    1
-                )
-            )
-            = 4,
-            'success',
-            max(
-                multiIf(
-                    s.status = 'success',
-                    4,
-                    s.status = 'no_packages_found',
-                    3,
-                    s.status = 'failure',
-                    2,
-                    1
-                )
-            )
-            = 3,
-            'no_packages_found',
-            max(
-                multiIf(
-                    s.status = 'success',
-                    4,
-                    s.status = 'no_packages_found',
-                    3,
-                    s.status = 'failure',
-                    2,
-                    1
-                )
-            )
-            = 2,
-            'failure',
-            'running'
-        ) as lastScanStatus
-    FROM vulnerability_scans as s FINAL
-    WHERE
-        s.repoUrl in (select arrayJoin(repositories) from segments_filtered)
-        {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %}
-
 NODE summary
 SQL >
     %
@@ -62,7 +11,3 @@ SQL >
     WHERE
         repoUrl in (select arrayJoin(repositories) from segments_filtered)
         {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %}
-
-NODE vulnerabilities_summary_result
-SQL >
-    SELECT summary.*, last_scan_status.lastScanStatus FROM summary CROSS JOIN last_scan_status