Merge pull request #1841 from jetstreamapp/bug/max-query-offset

fix: reduce number of objects included in query

Author: paustint

libs/features/manage-permissions/src/usePermissionRecords.tsx

@@ -1,7 +1,7 @@
 import { logger } from '@jetstream/shared/client-logger';
 import { describeSObject, queryAll, queryAllUsingOffset } from '@jetstream/shared/data';
 import { tracker } from '@jetstream/shared/ui-utils';
-import { getErrorMessage, groupByFlat } from '@jetstream/shared/utils';
+import { getErrorMessage, groupByFlat, splitArrayToMaxSize } from '@jetstream/shared/utils';
 import {
   EntityParticlePermissionsRecord,
   FieldPermissionDefinitionMap,
@@ -133,21 +133,28 @@ export function usePermissionRecords(selectedOrg: SalesforceOrgUi, sobjects: str
   };
 }
 
+/**
+ * Number of queries to run concurrently. Offset-paged queries (EntityParticle) are split into many small batches to stay
+ * under Salesforce's OFFSET cap, so we run them in bounded waves rather than sequentially to keep large selections fast.
+ */
+const QUERY_CONCURRENCY = 5;
+
 // This could be eligible to pull into generic method for expanded use
 async function queryAndCombineResults<T>(
   selectedOrg: SalesforceOrgUi,
   queries: string[],
   useOffset = false,
   isTooling = false,
 ): Promise<T[]> {
-  let output: T[] = [];
-  for (const currQuery of queries) {
-    if (useOffset) {
-      const { queryResults } = await queryAllUsingOffset<T>(selectedOrg, currQuery, isTooling);
-      output = output.concat(queryResults.records);
-    } else {
-      const { queryResults } = await queryAll<T>(selectedOrg, currQuery, isTooling);
-      output = output.concat(queryResults.records);
+  const runQuery = (currQuery: string) =>
+    useOffset ? queryAllUsingOffset<T>(selectedOrg, currQuery, isTooling) : queryAll<T>(selectedOrg, currQuery, isTooling);
+
+  const output: T[] = [];
+  // Results are keyed/grouped downstream, so ordering does not matter - run each wave concurrently
+  for (const wave of splitArrayToMaxSize(queries, QUERY_CONCURRENCY)) {
+    const waveResults = await Promise.all(wave.map(runQuery));
+    for (const { queryResults } of waveResults) {
+      output.push(...queryResults.records);
     }
   }
   return output;

libs/features/manage-permissions/src/utils/permission-manager-utils.ts

@@ -32,6 +32,12 @@ import { Query, WhereClause, composeQuery, getField } from '@jetstreamapp/soql-p
 
 const MAX_OBJ_IN_QUERY = 100;
 
+/**
+ * EntityParticle does not support queryMore, so we page it using OFFSET, which Salesforce caps at 2000 for this object.
+ * Keep the objects really small to avoid hitting the 2000 limit
+ */
+const MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY = 2;
+
 export function filterPermissionsSobjects(sobject: DescribeGlobalSObjectResult | null) {
   if (!sobject) {
     return false;
@@ -569,7 +575,7 @@ export function permissionsHaveError<T extends PermissionDefinitionMap>(permissi
  * @returns query for all permissionable fields
  */
 export function getQueryForAllPermissionableFields(allSobjects: string[]): string[] {
-  const queries = splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_QUERY).map((sobjects) => {
+  const queries = splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY).map((sobjects) => {
     return composeQuery({
       fields: [
         getField('QualifiedApiName'),