chore: create new organizations datasource with inferred slug (IN-1174) (#4132)

Signed-off-by: Joana Maia <jmaia@contractor.linuxfoundation.org>

Author: joanagmaia

services/libs/tinybird/datasources/organizations_populated_slug.datasource

@@ -0,0 +1,38 @@
+DESCRIPTION >
+    Denormalized organizations datasource: every column from organizations FINAL plus a
+    deterministic globally unique slug derived from displayName and a total contributionCount
+    across all segments.
+    Only includes organizations that have at least one activity recorded in
+    cdp_organization_segment_aggregates_ds (activityCountState > 0).
+    Slug rules:
+    - slug = lowerUTF8(displayName) with non-alphanumeric runs collapsed to '-' and
+    leading/trailing dashes trimmed.
+    - If two or more organizations produce the same baseSlug, the one with the highest
+    contributionCount keeps the bare slug; the rest get '--1', '--2', ... suffixes
+    (double-dash) ordered by descending contributionCount, with id ASC as the
+    deterministic tiebreaker. Double-dash prevents collisions with natural slugs that
+    end in a digit or word (e.g. 'dark-1' vs 'dark--1').
+    - If the derived slug would be empty (displayName contains only non-ASCII or special
+    characters), the slug falls back to id.
+    Refreshed hourly at :35 by organizations_populate_slug.pipe.
+
+SCHEMA >
+    `id` String,
+    `displayName` String,
+    `location` String DEFAULT '',
+    `logo` String DEFAULT '',
+    `tags` Array(String) DEFAULT [],
+    `employees` UInt32 DEFAULT 0,
+    `createdAt` DateTime64(3),
+    `updatedAt` DateTime64(3),
+    `isTeamOrganization` UInt8 DEFAULT 0,
+    `type` String DEFAULT '',
+    `size` String DEFAULT '',
+    `headline` String DEFAULT '',
+    `industry` String DEFAULT '',
+    `founded` UInt16 DEFAULT 0,
+    `slug` String,
+    `contributionCount` UInt64
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY id

services/libs/tinybird/pipes/organizations_populate_slug.pipe

@@ -0,0 +1,123 @@
+DESCRIPTION >
+    Populates organizations_populated_slug with one row per organization that has contributions.
+    Carries all columns from organizations FINAL and adds a deterministic globally unique
+    slug and a total contributionCount (sourced from cdp_organization_segment_aggregates_ds).
+
+NODE organizations_slugged_contribution_counts
+DESCRIPTION >
+    Total activity count per organization summed across all segments, read from the
+    pre-aggregated AggregatingMergeTree. Much faster than scanning the bucket union.
+
+SQL >
+    SELECT organizationId, countMerge(activityCountState) AS contributionCount
+    FROM cdp_organization_segment_aggregates_ds
+    WHERE organizationId != ''
+    GROUP BY organizationId
+
+NODE organizations_slugged_enriched
+DESCRIPTION >
+    Join contribution counts with full org metadata and compute the base slug candidate.
+
+SQL >
+    SELECT
+        org.id AS id,
+        org.displayName AS displayName,
+        org.location AS location,
+        org.logo AS logo,
+        org.tags AS tags,
+        org.employees AS employees,
+        org.createdAt AS createdAt,
+        org.updatedAt AS updatedAt,
+        org.isTeamOrganization AS isTeamOrganization,
+        org.type AS type,
+        org.size AS size,
+        org.headline AS headline,
+        org.industry AS industry,
+        org.founded AS founded,
+        c.contributionCount AS contributionCount,
+        replaceRegexpAll(
+            replaceRegexpAll(lowerUTF8(org.displayName), '[^a-z0-9]+', '-'), '(^-+)|(-+$)', ''
+        ) AS generatedSlug
+    FROM organizations_slugged_contribution_counts c
+    INNER JOIN organizations org FINAL ON org.id = c.organizationId
+
+NODE organizations_slugged_base_slug
+DESCRIPTION >
+    Apply empty-slug fallback: if generatedSlug is empty, use the org id as baseSlug.
+
+SQL >
+    SELECT
+        id,
+        displayName,
+        location,
+        logo,
+        tags,
+        employees,
+        createdAt,
+        updatedAt,
+        isTeamOrganization,
+        type,
+        size,
+        headline,
+        industry,
+        founded,
+        contributionCount,
+        if(length(generatedSlug) > 0, generatedSlug, id) AS baseSlug
+    FROM organizations_slugged_enriched
+
+NODE organizations_slugged_ranked
+DESCRIPTION >
+    Assign final globally unique slugs. Orgs with a unique baseSlug keep it as-is.
+    Orgs in a collision group (>=2 share the same baseSlug) all get a '--N' suffix
+    ordered by descending contributionCount (id ASC tiebreaker). Double-dash is used
+    so synthetic suffixes never collide with natural slugs that end in a number or word
+    (e.g. 'dark-1' from 'Dark-1' vs 'dark--1' from 'Dark++' in a collision group).
+
+SQL >
+    SELECT
+        id,
+        displayName,
+        location,
+        logo,
+        tags,
+        employees,
+        createdAt,
+        updatedAt,
+        isTeamOrganization,
+        type,
+        size,
+        headline,
+        industry,
+        founded,
+        if(groupSize = 1 OR slugRank = 1, baseSlug, concat(baseSlug, '--', toString(slugRank))) AS slug,
+        contributionCount
+    FROM
+        (
+            SELECT
+                id,
+                displayName,
+                location,
+                logo,
+                tags,
+                employees,
+                createdAt,
+                updatedAt,
+                isTeamOrganization,
+                type,
+                size,
+                headline,
+                industry,
+                founded,
+                contributionCount,
+                baseSlug,
+                count() OVER (PARTITION BY baseSlug) AS groupSize,
+                row_number() OVER (
+                    PARTITION BY baseSlug ORDER BY contributionCount DESC, id ASC
+                ) AS slugRank
+            FROM organizations_slugged_base_slug
+        )
+
+TYPE COPY
+TARGET_DATASOURCE organizations_populated_slug
+COPY_MODE replace
+COPY_SCHEDULE 35 * * * *