perf: optimize enrichable members selection query (#4389)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts

@@ -137,27 +137,48 @@ export async function fetchMembersForEnrichment(
           SELECT 1 FROM "memberEnrichmentCache" mec
           WHERE mec."memberId" = members.id
           AND mec.source = '${input.source}'
-          AND EXTRACT(EPOCH FROM (now() - mec."updatedAt")) < ${input.cacheObsoleteAfterSeconds})
+          AND mec."updatedAt" > now() - make_interval(secs => ${input.cacheObsoleteAfterSeconds}))
       )`,
       )
     }
 
     enrichableBySqlConditions.push(`(${input.enrichableBySql})`)
   })
 
-  let enrichableBySqlJoined = ''
-
-  if (enrichableBySqlConditions.length > 0) {
-    enrichableBySqlJoined = `(${enrichableBySqlConditions.join(' OR ')}) `
-  }
+  const enrichableBySqlJoined =
+    enrichableBySqlConditions.length > 0 ? `(${enrichableBySqlConditions.join(' OR ')})` : 'TRUE'
 
+  // Pick top-N by activity first, then load identities for those rows only.
   return db.connection().query(
     `
+    WITH candidates AS (
+      SELECT
+           members.id,
+           members."displayName",
+           members.attributes->'location'->>'default' AS location,
+           members.attributes->'websiteUrl'->>'default' AS website,
+           coalesce("membersGlobalActivityCount".total_count, 0) AS "activityCount"
+      FROM "membersGlobalActivityCount"
+           INNER JOIN members ON members.id = "membersGlobalActivityCount"."memberId"
+      WHERE members."deletedAt" IS NULL
+        AND coalesce((members.attributes ->'isBot'->>'default')::boolean, false) = false
+        AND coalesce((members.attributes ->'isOrganization'->>'default')::boolean, false) = false
+        AND EXISTS (
+          SELECT 1
+          FROM "memberIdentities" mi
+          WHERE mi."memberId" = members.id
+            AND mi."deletedAt" IS NULL
+            AND ${enrichableBySqlJoined}
+        )
+        AND (${cacheAgeInnerQueryItems.join(' OR ')})
+      ORDER BY "membersGlobalActivityCount".total_count DESC
+      LIMIT $1
+    )
     SELECT
-         members."id",
-         members."displayName",
-         members.attributes->'location'->>'default' AS location,
-         members.attributes->'websiteUrl'->>'default' AS website,
+         c.id,
+         c."displayName",
+         c.location,
+         c.website,
          JSON_AGG(
            JSON_BUILD_OBJECT(
              'platform', mi.platform,
@@ -166,19 +187,18 @@ export async function fetchMembersForEnrichment(
              'verified', mi.verified
            )
          ) AS identities,
-         MAX(coalesce("membersGlobalActivityCount".total_count, 0)) AS "activityCount"
-    FROM members
-         INNER JOIN "memberIdentities" mi ON mi."memberId" = members.id and mi."deletedAt" is null
-         LEFT JOIN "membersGlobalActivityCount" ON "membersGlobalActivityCount"."memberId" = members.id
-    WHERE
-      ${enrichableBySqlJoined}
-      AND coalesce((members.attributes ->'isBot'->>'default')::boolean, false) = false 
-      AND coalesce((members.attributes ->'isOrganization'->>'default')::boolean, false) = false
-      AND members."deletedAt" IS NULL
-      AND (${cacheAgeInnerQueryItems.join(' OR ')})
-    GROUP BY members.id
-    ORDER BY "activityCount" DESC
-    LIMIT $1;
+         c."activityCount"
+    FROM candidates c
+         INNER JOIN members ON members.id = c.id
+         INNER JOIN "memberIdentities" mi
+           ON mi."memberId" = c.id
+          AND mi."deletedAt" IS NULL
+         CROSS JOIN LATERAL (
+           SELECT c."activityCount" AS total_count
+         ) AS "membersGlobalActivityCount"
+    WHERE ${enrichableBySqlJoined}
+    GROUP BY c.id, c."displayName", c.location, c.website, c."activityCount"
+    ORDER BY c."activityCount" DESC;
     `,
     [limit],
   )