feat: add adaptive target query threshold

- add ScriptObject.targetFullQueryRecordsThreshold.

- choose target full query by effective target record count.

- keep queryAllTarget, complex/autonumber IDs, and special objects full query.

- add schema and unit coverage for target query strategy.

Author: hknokh2

schemas/export.schema.json

@@ -286,6 +286,10 @@
         "queryAllTarget": {
           "type": "boolean"
         },
+        "targetFullQueryRecordsThreshold": {
+          "type": "integer",
+          "minimum": 0
+        },
         "skipExistingRecords": {
           "type": "boolean"
         },

src/modules/constants/Constants.ts

@@ -130,6 +130,7 @@ export const DEFAULT_MAX_PARALLEL_BLOB_DOWNLOADS = 20;
 export const DEFAULT_MAX_PARALLEL_EXEC_TASKS = 10;
 export const MAX_FETCH_SIZE = 100_000;
 export const QUERY_BULK_API_THRESHOLD = 30_000;
+export const TARGET_FULL_QUERY_RECORDS_THRESHOLD = 30_000;
 export const BULK_API_V2_BLOCK_SIZE = 1000;
 export const BULK_API_V2_MAX_CSV_SIZE_IN_BYTES = 145_000_000;
 export const POLL_TIMEOUT = 3_000_000;

src/modules/models/job/MigrationJob.ts

@@ -36,6 +36,7 @@ import {
   GROUP_OBJECT_NAME,
   VALUE_MAPPING_CSV_FILENAME,
   USER_AND_GROUP_FILENAME,
+  TARGET_FULL_QUERY_RECORDS_THRESHOLD,
 } from '../../constants/Constants.js';
 import type { LoggerType } from '../../logging/LoggerType.js';
 import MappingResolver from '../../mapping/MappingResolver.js';
@@ -454,6 +455,7 @@ export default class MigrationJob implements ISFdmuRunCustomAddonJob {
     this._createTasks();
     this.tasks.forEach((task) => task.refreshPreflightState());
     await this._calculateRecordCountsAsync();
+    this._resolveTargetQueryStrategies();
   }
 
   /**
@@ -2122,19 +2124,44 @@ export default class MigrationJob implements ISFdmuRunCustomAddonJob {
 
       const shouldProcessAll =
         Boolean(updatedObject.master) || updatedObject.isSpecialObject || updatedObject.isObjectWithoutRelationships;
-      if (shouldProcessAll) {
-        updatedObject.processAllSource = true;
-        updatedObject.processAllTarget = true;
-      } else {
-        updatedObject.processAllSource = false;
-        updatedObject.processAllTarget = updatedObject.hasComplexExternalId || updatedObject.hasAutonumberExternalId;
-      }
+      updatedObject.processAllSource = shouldProcessAll;
+      updatedObject.processAllTarget = this._requiresFullTargetQuery(updatedObject, 0);
 
       map.set(updatedObject, task);
     });
     return map;
   }
 
+  /**
+   * Resolves target query strategies after target record counts are available.
+   */
+  private _resolveTargetQueryStrategies(): void {
+    this.tasks.forEach((task) => {
+      const targetTotalRecords = task.targetData.totalRecordCount;
+      const { scriptObject } = task;
+      scriptObject.processAllTarget = this._requiresFullTargetQuery(scriptObject, targetTotalRecords);
+    });
+  }
+
+  /**
+   * Returns true when target records must be read with the full target query.
+   *
+   * @param object - Script object to inspect.
+   * @param targetTotalRecords - Effective target record count for the object query.
+   * @returns True when the full target query should be used.
+   */
+  private _requiresFullTargetQuery(object: ScriptObject, targetTotalRecords: number): boolean {
+    void this;
+    const threshold = object.targetFullQueryRecordsThreshold ?? TARGET_FULL_QUERY_RECORDS_THRESHOLD;
+    return (
+      object.queryAllTarget ||
+      object.hasComplexExternalId ||
+      object.hasAutonumberExternalId ||
+      object.isSpecialObject ||
+      targetTotalRecords < threshold
+    );
+  }
+
   /**
    * Builds an ordered task chain using the legacy algorithm.
    *

src/modules/models/script/ScriptObject.ts

@@ -278,6 +278,11 @@ export default class ScriptObject {
    */
   public queryAllTarget = false;
 
+  /**
+   * Target record count threshold for using full target query instead of filtered IN queries.
+   */
+  public targetFullQueryRecordsThreshold: number | undefined;
+
   /**
    * Skips source records that already exist in target.
    */

test/modules/job/migration-job.test.ts

@@ -11,6 +11,7 @@ import { Common } from '../../../src/modules/common/Common.js';
 import LoggingContext from '../../../src/modules/logging/LoggingContext.js';
 import LoggingService from '../../../src/modules/logging/LoggingService.js';
 import { ADDON_EVENTS, DATA_MEDIA_TYPE, OPERATION } from '../../../src/modules/common/Enumerations.js';
+import { TARGET_FULL_QUERY_RECORDS_THRESHOLD } from '../../../src/modules/constants/Constants.js';
 import { UnresolvableWarning } from '../../../src/modules/models/common/UnresolvableWarning.js';
 import MigrationJob from '../../../src/modules/models/job/MigrationJob.js';
 import type { IMetadataProvider } from '../../../src/modules/models/job/IMetadataProvider.js';
@@ -22,6 +23,7 @@ import ScriptOrg from '../../../src/modules/models/script/ScriptOrg.js';
 import SFieldDescribe from '../../../src/modules/models/sf/SFieldDescribe.js';
 import SObjectDescribe from '../../../src/modules/models/sf/SObjectDescribe.js';
 import OrgConnectionAdapter from '../../../src/modules/org/OrgConnectionAdapter.js';
+import OrgDataService from '../../../src/modules/org/OrgDataService.js';
 
 type FieldInitType = Partial<SFieldDescribe> & { name: string };
 type OrgConnectionStubType = {
@@ -99,6 +101,162 @@ describe('MigrationJob', () => {
     Common.logger = originalLogger;
   });
 
+  const prepareSingleObjectStrategyJobAsync = async (
+    object: ScriptObject,
+    describe: SObjectDescribe,
+    targetCount: number
+  ): Promise<MigrationJob> => {
+    const script = new Script();
+    script.objectSets = [new ScriptObjectSet([object])];
+
+    const sourceOrg = new ScriptOrg();
+    sourceOrg.name = 'source';
+    sourceOrg.media = DATA_MEDIA_TYPE.Org;
+    sourceOrg.script = script;
+    const targetOrg = new ScriptOrg();
+    targetOrg.name = 'target';
+    targetOrg.media = DATA_MEDIA_TYPE.Org;
+    targetOrg.script = script;
+    script.sourceOrg = sourceOrg;
+    script.targetOrg = targetOrg;
+
+    const metadataProvider = createMetadataProvider(new Map([[object.name, describe]]));
+    const job = new MigrationJob({ script, metadataProvider });
+    const originalConnection = OrgConnectionAdapter.getConnectionForAliasAsync.bind(OrgConnectionAdapter);
+    const queryOrgAsyncDescriptor = Object.getOwnPropertyDescriptor(OrgDataService.prototype, 'queryOrgAsync');
+    if (typeof queryOrgAsyncDescriptor?.value !== 'function') {
+      throw new Error('OrgDataService.queryOrgAsync descriptor was not found.');
+    }
+    const originalQueryOrgAsync = queryOrgAsyncDescriptor.value as OrgDataService['queryOrgAsync'];
+    OrgConnectionAdapter.getConnectionForAliasAsync = async () => createOrgConnectionStub() as never;
+    OrgDataService.prototype.queryOrgAsync = async (
+      query: string,
+      org: ScriptOrg
+    ): Promise<Array<Record<string, unknown>>> => {
+      if (!query.includes('COUNT')) {
+        return [];
+      }
+      return [{ CNT: org.name === 'target' ? targetCount : 1 }];
+    };
+
+    try {
+      await job.loadAsync();
+      await job.setupAsync();
+      await job.prepareAsync();
+    } finally {
+      OrgConnectionAdapter.getConnectionForAliasAsync = originalConnection;
+      OrgDataService.prototype.queryOrgAsync = originalQueryOrgAsync;
+    }
+
+    return job;
+  };
+
+  it('uses filtered target queries for master objects with simple external id above the target threshold', async () => {
+    const account = new ScriptObject('Account');
+    account.operation = OPERATION.Upsert;
+    account.query = 'SELECT Id, External_Key__c FROM Account';
+    account.externalId = 'External_Key__c';
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      account,
+      createDescribe('Account', [{ name: 'Id' }, { name: 'External_Key__c', updateable: true, creatable: true }]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD + 1
+    );
+
+    const task = job.getTaskBySObjectName('Account');
+    assert.equal(task?.scriptObject.processAllSource, true);
+    assert.equal(task?.scriptObject.processAllTarget, false);
+  });
+
+  it('uses full target query for simple external id below the target threshold', async () => {
+    const account = new ScriptObject('Account');
+    account.operation = OPERATION.Upsert;
+    account.query = 'SELECT Id, External_Key__c FROM Account';
+    account.externalId = 'External_Key__c';
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      account,
+      createDescribe('Account', [{ name: 'Id' }, { name: 'External_Key__c', updateable: true, creatable: true }]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD - 1
+    );
+
+    const task = job.getTaskBySObjectName('Account');
+    assert.equal(task?.scriptObject.processAllTarget, true);
+  });
+
+  it('uses the object target full-query threshold override', async () => {
+    const account = new ScriptObject('Account');
+    account.operation = OPERATION.Upsert;
+    account.query = 'SELECT Id, External_Key__c FROM Account';
+    account.externalId = 'External_Key__c';
+    account.targetFullQueryRecordsThreshold = TARGET_FULL_QUERY_RECORDS_THRESHOLD + 10;
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      account,
+      createDescribe('Account', [{ name: 'Id' }, { name: 'External_Key__c', updateable: true, creatable: true }]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD + 1
+    );
+
+    const task = job.getTaskBySObjectName('Account');
+    assert.equal(task?.scriptObject.processAllTarget, true);
+  });
+
+  it('keeps queryAllTarget on full target query regardless of target count', async () => {
+    const account = new ScriptObject('Account');
+    account.operation = OPERATION.Upsert;
+    account.query = 'SELECT Id, External_Key__c FROM Account';
+    account.externalId = 'External_Key__c';
+    account.queryAllTarget = true;
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      account,
+      createDescribe('Account', [{ name: 'Id' }, { name: 'External_Key__c', updateable: true, creatable: true }]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD + 1
+    );
+
+    const task = job.getTaskBySObjectName('Account');
+    assert.equal(task?.scriptObject.processAllTarget, true);
+  });
+
+  it('uses full target query for complex external id regardless of target count', async () => {
+    const account = new ScriptObject('Account');
+    account.operation = OPERATION.Upsert;
+    account.query = 'SELECT Id, External_Key__c, Name FROM Account';
+    account.externalId = 'External_Key__c;Name';
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      account,
+      createDescribe('Account', [
+        { name: 'Id' },
+        { name: 'External_Key__c', updateable: true, creatable: true },
+        { name: 'Name', nameField: true, updateable: true, creatable: true },
+      ]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD + 1
+    );
+
+    const task = job.getTaskBySObjectName('Account');
+    assert.equal(task?.scriptObject.processAllTarget, true);
+  });
+
+  it('uses full target query for special objects regardless of target count', async () => {
+    const recordType = new ScriptObject('RecordType');
+    recordType.query = 'SELECT Id, DeveloperName, NamespacePrefix, SobjectType FROM RecordType';
+
+    const job = await prepareSingleObjectStrategyJobAsync(
+      recordType,
+      createDescribe('RecordType', [
+        { name: 'Id' },
+        { name: 'DeveloperName' },
+        { name: 'NamespacePrefix' },
+        { name: 'SobjectType' },
+      ]),
+      TARGET_FULL_QUERY_RECORDS_THRESHOLD + 1
+    );
+
+    const task = job.getTaskBySObjectName('RecordType');
+    assert.equal(task?.scriptObject.processAllTarget, true);
+  });
+
   it('orders tasks by RecordType, readonly, and lookup parents', async () => {
     const recordType = new ScriptObject('RecordType');
     recordType.operation = OPERATION.Readonly;

test/modules/script/script-loader.test.ts

@@ -234,6 +234,31 @@ describe('ScriptLoader', () => {
     }
   });
 
+  it('loads object target full-query threshold from export.json', async () => {
+    const rootPath = createTempDir();
+    const payload: ScriptPayloadType = {
+      objects: [
+        {
+          query: 'SELECT Id, Name FROM Account',
+          operation: 'Upsert',
+          externalId: 'Name',
+          targetFullQueryRecordsThreshold: 125,
+        },
+      ],
+    };
+    writeExportJson(rootPath, payload);
+    const logger = createLogger(rootPath);
+
+    try {
+      const script = await ScriptLoader.loadFromPathAsync(rootPath, logger);
+      const object = script.objectSets[0].objects[0];
+
+      assert.equal(object.targetFullQueryRecordsThreshold, 125);
+    } finally {
+      fs.rmSync(rootPath, { recursive: true, force: true });
+    }
+  });
+
   it('preserves explicit null or empty values from export.json', async () => {
     const rootPath = createTempDir();
     const previousReadDelimiter = Common.csvReadFileDelimiter;