perf: use hasMore instead of live count on merge suggestion lists (#4261)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

backend/src/database/repositories/memberRepository.ts

@@ -410,6 +410,9 @@ class MemberRepository {
       return { count: await getTotalCount() }
     }
 
+    const pageLimit = args.limit
+    const queryLimit = pageLimit + 1
+
     const mems = await options.database.sequelize.query(
       `
         SELECT
@@ -452,7 +455,7 @@ class MemberRepository {
       {
         replacements: {
           segmentIds,
-          limit: args.limit,
+          limit: queryLimit,
           offset: args.offset,
           displayName: args?.filter?.displayName ? `${args.filter.displayName}%` : undefined,
           memberId: args?.filter?.memberId,
@@ -463,7 +466,10 @@ class MemberRepository {
       },
     )
 
-    if (mems.length > 0) {
+    const hasMore = mems.length > pageLimit
+    const pageRows = hasMore ? mems.slice(0, pageLimit) : mems
+
+    if (pageRows.length > 0) {
       let result
 
       if (args.detail) {
@@ -522,7 +528,7 @@ class MemberRepository {
           }
         }
 
-        for (const mem of mems) {
+        for (const mem of pageRows) {
           memberPromises.push(findMemberInfo(mem.id))
           toMergePromises.push(findMemberInfo(mem.toMergeId))
         }
@@ -532,10 +538,10 @@ class MemberRepository {
 
         result = memberResults.map((i, idx) => ({
           members: [i, memberToMergeResults[idx]],
-          similarity: mems[idx].similarity,
+          similarity: pageRows[idx].similarity,
         }))
       } else {
-        result = mems.map((i) => ({
+        result = pageRows.map((i) => ({
           members: [
             {
               id: i.id,
@@ -554,12 +560,12 @@ class MemberRepository {
         }))
       }
 
-      return { rows: result, count: await getTotalCount(), limit: args.limit, offset: args.offset }
+      return { rows: result, hasMore, limit: args.limit, offset: args.offset }
     }
 
     return {
       rows: [{ members: [], similarity: 0 }],
-      count: await getTotalCount(),
+      hasMore: false,
       limit: args.limit,
       offset: args.offset,
     }

backend/src/database/repositories/organizationRepository.ts

@@ -944,6 +944,9 @@ class OrganizationRepository {
       return { count: await getTotalCount() }
     }
 
+    const pageLimit = args.limit
+    const queryLimit = pageLimit + 1
+
     const orgs = await options.database.sequelize.query(
       `
         SELECT
@@ -990,7 +993,7 @@ class OrganizationRepository {
       {
         replacements: {
           segmentIds,
-          limit: args.limit,
+          limit: queryLimit,
           offset: args.offset,
           displayName: args?.filter?.displayName ? `${args.filter.displayName}%` : undefined,
           mergeActionType: MergeActionType.ORG,
@@ -1001,14 +1004,17 @@ class OrganizationRepository {
       },
     )
 
-    if (orgs.length > 0) {
+    const hasMore = orgs.length > pageLimit
+    const pageRows = hasMore ? orgs.slice(0, pageLimit) : orgs
+
+    if (pageRows.length > 0) {
       let result
 
       if (args.detail) {
         const organizationPromises = []
         const toMergePromises = []
 
-        for (const org of orgs) {
+        for (const org of pageRows) {
           organizationPromises.push(
             OrganizationRepository.findById(org.id, options, org.primarySegmentId),
           )
@@ -1022,10 +1028,10 @@ class OrganizationRepository {
 
         result = organizationResults.map((i, idx) => ({
           organizations: [i, organizationToMergeResults[idx]],
-          similarity: orgs[idx].similarity,
+          similarity: pageRows[idx].similarity,
         }))
       } else {
-        result = orgs.map((o) => ({
+        result = pageRows.map((o) => ({
           organizations: [
             {
               id: o.id,
@@ -1055,15 +1061,15 @@ class OrganizationRepository {
 
       return {
         rows: result,
-        count: await getTotalCount(),
+        hasMore,
         limit: args.limit,
         offset: args.offset,
       }
     }
 
     return {
       rows: [{ organizations: [], similarity: 0 }],
-      count: await getTotalCount(),
+      hasMore: false,
       limit: args.limit,
       offset: args.offset,
     }

frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions.vue

@@ -18,7 +18,7 @@
           </div>
         </template>
       </lf-data-quality-member-merge-suggestions-item>
-      <div v-if="mergeSuggestions.length < total" class="pt-4">
+      <div v-if="hasMore" class="pt-4">
         <lf-button
           type="primary-ghost"
           loading-text="Loading suggestions..."
@@ -73,7 +73,7 @@ const props = defineProps<{
 const loading = ref(true);
 const limit = ref(20);
 const offset = ref(0);
-const total = ref(0);
+const hasMore = ref(false);
 const mergeSuggestions = ref<any[]>([]);
 
 const isModalOpen = ref<boolean>(false);
@@ -91,7 +91,7 @@ const loadMergeSuggestions = () => {
     segments: segments.value,
   })
     .then((res) => {
-      total.value = +res.count;
+      hasMore.value = Boolean(res.hasMore);
       const rows = res.rows.filter((s: any) => s.similarity > 0);
       if (+res.offset > 0) {
         mergeSuggestions.value = [...mergeSuggestions.value, ...rows];

frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions.vue

@@ -15,7 +15,7 @@
           </lf-button>
         </template>
       </lf-data-quality-organization-merge-suggestions-item>
-      <div v-if="mergeSuggestions.length < total" class="pt-4">
+      <div v-if="hasMore" class="pt-4">
         <lf-button
           type="primary-ghost"
           loading-text="Loading suggestions..."
@@ -67,7 +67,7 @@ const props = defineProps<{
 const loading = ref(true);
 const limit = ref(20);
 const offset = ref(0);
-const total = ref(0);
+const hasMore = ref(false);
 const mergeSuggestions = ref<any[]>([]);
 
 const isModalOpen = ref<boolean>(false);
@@ -84,7 +84,7 @@ const loadMergeSuggestions = () => {
     orderBy: ['similarity_DESC', 'activityCount_DESC'],
   })
     .then((res) => {
-      total.value = +res.count;
+      hasMore.value = Boolean(res.hasMore);
       const rows = res.rows.filter((s: any) => s.similarity > 0);
       if (+res.offset > 0) {
         mergeSuggestions.value = [...mergeSuggestions.value, ...rows];

frontend/src/modules/member/components/member-merge-suggestions.vue

@@ -7,7 +7,7 @@
           <lf-button
             type="secondary"
             size="small"
-            :disabled="loading || offset <= 0 || count === 0"
+            :disabled="loading || offset <= 0 || !hasSuggestion"
             :icon-only="true"
             @click="fetch(offset - 1)"
           >
@@ -16,7 +16,7 @@
           <lf-button
             type="secondary"
             size="small"
-            :disabled="loading || offset >= count - 1 || count === 0"
+            :disabled="loading || !hasMore"
             :icon-only="true"
             @click="fetch(offset + 1)"
           >
@@ -26,16 +26,10 @@
 
         <app-loading v-if="loading" height="16px" width="128px" radius="3px" />
         <div
-          v-else-if="Math.ceil(count) > 1"
+          v-else-if="hasSuggestion"
           class="text-xs leading-5 text-gray-500"
         >
-          <div>{{ offset + 1 }} of {{ Math.ceil(count) }} suggestions</div>
-        </div>
-        <div
-          v-else-if="Math.ceil(count) === 1"
-          class="text-xs leading-5 text-gray-500"
-        >
-          <div>1 suggestion</div>
+          <div>Suggestion {{ offset + 1 }}</div>
         </div>
         <div
           v-else
@@ -48,15 +42,15 @@
         <app-member-merge-similarity v-if="!loading && membersToMerge.similarity" :similarity="membersToMerge.similarity" />
         <lf-button
           type="secondary"
-          :disabled="loading || isEditLockedForSampleData || count === 0"
+          :disabled="loading || isEditLockedForSampleData || !hasSuggestion"
           :loading="sendingIgnore"
           @click="ignoreSuggestion()"
         >
           Ignore suggestion
         </lf-button>
         <lf-button
           type="primary"
-          :disabled="loading || isEditLockedForSampleData || count === 0"
+          :disabled="loading || isEditLockedForSampleData || !hasSuggestion"
           :loading="sendingMerge"
           @click="mergeSuggestion()"
         >
@@ -66,7 +60,7 @@
       </div>
     </header>
 
-    <div v-if="loading || count > 0">
+    <div v-if="loading || hasSuggestion">
       <!-- Comparison -->
       <!-- Loading -->
       <div v-if="loading" class="flex p-5">
@@ -170,7 +164,8 @@ const { getContributorMergeActions } = useContributorStore();
 const membersToMerge = ref([]);
 const primary = ref(0);
 const offset = ref(props.offset);
-const count = ref(0);
+const hasMore = ref(false);
+const hasSuggestion = ref(false);
 const loading = ref(false);
 
 const sendingIgnore = ref(false);
@@ -204,25 +199,45 @@ const preview = computed(() => {
   return mergedMembers;
 });
 
-const fetch = (page) => {
+const updateSuggestionsState = (res) => {
+  hasMore.value = Boolean(res.hasMore);
+  const rows = res.rows.filter((suggestion) => suggestion.similarity > 0);
+
+  if (rows.length > 0) {
+    hasSuggestion.value = true;
+    [membersToMerge.value] = rows;
+  } else {
+    hasSuggestion.value = false;
+    membersToMerge.value = [];
+  }
+};
+
+const fetch = (page, trackNavigation = true) => {
   if (page > -1) {
     offset.value = page;
   }
 
-  trackEvent({
-    key: FeatureEventKey.NAVIGATE_MEMBERS_MERGE_SUGGESTIONS,
-    type: EventType.FEATURE,
-  });
+  if (trackNavigation) {
+    trackEvent({
+      key: FeatureEventKey.NAVIGATE_MEMBERS_MERGE_SUGGESTIONS,
+      type: EventType.FEATURE,
+    });
+  }
 
   loading.value = true;
 
-  MemberService.fetchMergeSuggestions(1, offset.value, props.query ?? {})
+  return MemberService.fetchMergeSuggestions(1, offset.value, props.query ?? {})
     .then((res) => {
       offset.value = +res.offset;
-      count.value = res.count;
-      [membersToMerge.value] = res.rows;
+
+      updateSuggestionsState(res);
+
+      if (!hasSuggestion.value && offset.value > 0) {
+        return fetch(offset.value - 1, false);
+      }
 
       primary.value = 0;
+      return undefined;
     })
     .catch(() => {
       ToastStore.error(
@@ -252,8 +267,7 @@ const ignoreSuggestion = () => {
     .then(() => {
       ToastStore.success('Merging suggestion ignored successfully');
       getContributorMergeActions();
-      const nextIndex = offset.value >= (count.value - 1) ? Math.max(count.value - 2, 0) : offset.value;
-      fetch(nextIndex);
+      fetch(offset.value);
       changed.value = true;
     })
     .catch((error) => {
@@ -304,8 +318,7 @@ const mergeSuggestion = () => {
       );
       primary.value = 0;
 
-      const nextIndex = offset.value >= (count.value - 1) ? Math.max(count.value - 2, 0) : offset.value;
-      fetch(nextIndex);
+      fetch(offset.value);
       changed.value = true;
     })
     .catch((error) => {

frontend/src/modules/member/pages/member-merge-suggestions-page.vue

@@ -114,7 +114,7 @@
         </p>
       </div>
 
-      <div v-if="total > mergeSuggestions.length" class="mt-6 flex justify-center">
+      <div v-if="hasMore" class="mt-6 flex justify-center">
         <lf-button type="primary-ghost" size="small" :loading="loading" @click="loadMore()">
           <lf-icon name="arrow-down" />Load more
         </lf-button>
@@ -158,7 +158,7 @@ const mergeSuggestions = ref<any[]>([]);
 
 const isModalOpen = ref<boolean>(false);
 
-const total = ref<number>(0);
+const hasMore = ref<boolean>(false);
 const limit = ref<number>(10);
 const page = ref<number>(1);
 const loading = ref<boolean>(false);
@@ -187,7 +187,7 @@ const loadMergeSuggestions = (sort: boolean = false) => {
     detail: false,
   })
     .then((res) => {
-      total.value = +res.count;
+      hasMore.value = Boolean(res.hasMore);
       const rows = res.rows.filter((s: any) => s.similarity > 0);
       if (+res.offset > 0) {
         mergeSuggestions.value = [...mergeSuggestions.value, ...rows];