feat: add filtered query IN-clause field controls

- add ScriptObject.includedInClauseFields

- add ScriptObject.excludedFromInClauseFields

- keep denied lookup fields selected and exported

- update schema, loader, planner, and unit tests

Author: hknokh2

schemas/export.schema.json

@@ -323,6 +323,18 @@
             "type": "string"
           }
         },
+        "includedInClauseFields": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "excludedFromInClauseFields": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
         "fieldMapping": {
           "type": "array",
           "items": {

src/modules/models/job/MigrationJobTask.ts

@@ -3974,6 +3974,7 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
           if (
             field.isSimpleReference &&
             field.parentLookupObject &&
+            this._isFieldAllowedForInClause(field) &&
             !excludedObjects.includes(field.referencedObjectType.toLowerCase())
           ) {
             const parentTask = this.job.getTaskBySObjectName(field.parentLookupObject.name);
@@ -4025,6 +4026,25 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
     return queries;
   }
 
+  /**
+   * Checks whether a lookup field can drive filtered IN-clause source queries.
+   *
+   * @param field - Lookup field metadata.
+   * @returns True when field is allowed as a filtered-query pivot.
+   */
+  private _isFieldAllowedForInClause(field: SFieldDescribe): boolean {
+    const normalize = (fieldName: string): string => fieldName.trim().toLowerCase();
+    const fieldNames = new Set([field.name, field.nameId].map(normalize).filter((fieldName) => fieldName.length > 0));
+    const included = new Set(this.scriptObject.includedInClauseFields.map(normalize));
+    const excluded = new Set(this.scriptObject.excludedFromInClauseFields.map(normalize));
+    const hasIncluded = included.size > 0;
+    const isIncluded = !hasIncluded || [...fieldNames].some((fieldName) => included.has(fieldName));
+    if (!isIncluded) {
+      return false;
+    }
+    return ![...fieldNames].some((fieldName) => excluded.has(fieldName));
+  }
+
   /**
    * Returns the base query string from preflight output.
    *

src/modules/models/script/ScriptObject.ts

@@ -363,6 +363,16 @@ export default class ScriptObject {
    */
   public excludedFromUpdateFields: string[] = [];
 
+  /**
+   * Lookup fields explicitly allowed as filtered-query IN-clause pivots.
+   */
+  public includedInClauseFields: string[] = [];
+
+  /**
+   * Lookup fields excluded from filtered-query IN-clause pivots.
+   */
+  public excludedFromInClauseFields: string[] = [];
+
   /**
    * Runtime excluded fields from update operations.
    */

src/modules/script/ScriptLoader.ts

@@ -414,6 +414,22 @@ export default class ScriptLoader {
       },
       (value) => this._normalizeStringArray(value)
     );
+    this._assignArrayPropertyFromRaw<string>(
+      rawObject,
+      'includedInClauseFields',
+      (value) => {
+        object.includedInClauseFields = value as string[];
+      },
+      (value) => this._normalizeStringArray(value)
+    );
+    this._assignArrayPropertyFromRaw<string>(
+      rawObject,
+      'excludedFromInClauseFields',
+      (value) => {
+        object.excludedFromInClauseFields = value as string[];
+      },
+      (value) => this._normalizeStringArray(value)
+    );
     const operation = this._resolveOperation(rawObject.operation);
     if (typeof operation !== 'undefined') {
       object.operation = operation;

test/modules/job/migration-job-task.test.ts

@@ -36,6 +36,13 @@ type TaskFixtureType = {
   task: MigrationJobTask;
 };
 
+type InClauseFixtureType = {
+  childTask: MigrationJobTask;
+  parentTask: MigrationJobTask;
+  productTask: MigrationJobTask;
+  relatedProductTask: MigrationJobTask;
+};
+
 const createTempDir = (): string => fs.mkdtempSync(path.join(os.tmpdir(), 'sfdmu-task-'));
 
 type FieldInitType = Partial<SFieldDescribe> & { name: string };
@@ -72,6 +79,70 @@ const createTaskFixture = (): TaskFixtureType => {
 
 const createMap = (pairs: Array<[string, string]>): LookupIdMapType => new Map(pairs);
 
+const hasFilteredInClause = (queries: string[], fieldName: string): boolean =>
+  queries.some((query) => query.includes(`WHERE ${fieldName} IN (`) || query.includes(`AND ${fieldName} IN (`));
+
+const createInClauseFixture = (): InClauseFixtureType => {
+  const script = new Script();
+  const job = new MigrationJob({ script });
+  const parentObject = new ScriptObject('Parent_Offer__c');
+  const productObject = new ScriptObject('Product__c');
+  const relatedProductObject = new ScriptObject('Related_Product__c');
+  const childObject = new ScriptObject('Offer_Line_Item__c');
+  childObject.query = 'SELECT Id, Parent_Offer__c, Product__c, Related_Product__c FROM Offer_Line_Item__c';
+
+  const parentTask = new MigrationJobTask({ job, scriptObject: parentObject });
+  const productTask = new MigrationJobTask({ job, scriptObject: productObject });
+  const relatedProductTask = new MigrationJobTask({ job, scriptObject: relatedProductObject });
+  const childTask = new MigrationJobTask({ job, scriptObject: childObject });
+
+  job.tasks = [childTask, parentTask, productTask, relatedProductTask];
+  parentTask.sourceData.idRecordsMap.set('a01000000000001AAA', { Id: 'a01000000000001AAA' });
+  productTask.sourceData.idRecordsMap.set('a02000000000001AAA', { Id: 'a02000000000001AAA' });
+  relatedProductTask.sourceData.idRecordsMap.set('a03000000000001AAA', { Id: 'a03000000000001AAA' });
+
+  childTask.data.fieldsInQueryMap = new Map([
+    [
+      'Parent_Offer__c',
+      new SFieldDescribe({
+        name: 'Parent_Offer__c',
+        objectName: childObject.name,
+        type: 'reference',
+        lookup: true,
+        referencedObjectType: parentObject.name,
+        parentLookupObject: parentObject,
+        isDescribed: true,
+      }),
+    ],
+    [
+      'Product__c',
+      new SFieldDescribe({
+        name: 'Product__c',
+        objectName: childObject.name,
+        type: 'reference',
+        lookup: true,
+        referencedObjectType: productObject.name,
+        parentLookupObject: productObject,
+        isDescribed: true,
+      }),
+    ],
+    [
+      'Related_Product__c',
+      new SFieldDescribe({
+        name: 'Related_Product__c',
+        objectName: childObject.name,
+        type: 'reference',
+        lookup: true,
+        referencedObjectType: relatedProductObject.name,
+        parentLookupObject: relatedProductObject,
+        isDescribed: true,
+      }),
+    ],
+  ]);
+
+  return { childTask, parentTask, productTask, relatedProductTask };
+};
+
 describe('MigrationJobTask relationship external id matching', () => {
   it('links target records by nested relationship external id values', () => {
     const { task } = createTaskFixture();
@@ -97,6 +168,55 @@ describe('MigrationJobTask relationship external id matching', () => {
   });
 });
 
+describe('MigrationJobTask filtered IN-clause fields', () => {
+  it('uses every eligible lookup field when no IN-clause field filter is configured', () => {
+    const { childTask } = createInClauseFixture();
+
+    const queries = childTask.createFilteredQueries('forwards', false);
+
+    assert.equal(queries.length, 3);
+    assert.ok(hasFilteredInClause(queries, 'Parent_Offer__c'));
+    assert.ok(hasFilteredInClause(queries, 'Product__c'));
+    assert.ok(hasFilteredInClause(queries, 'Related_Product__c'));
+  });
+
+  it('uses only included lookup fields as filtered IN-clause pivots', () => {
+    const { childTask } = createInClauseFixture();
+    childTask.scriptObject.includedInClauseFields = ['Parent_Offer__c'];
+
+    const queries = childTask.createFilteredQueries('forwards', false);
+
+    assert.equal(queries.length, 1);
+    assert.ok(hasFilteredInClause(queries, 'Parent_Offer__c'));
+    assert.ok(!hasFilteredInClause(queries, 'Product__c'));
+    assert.ok(!hasFilteredInClause(queries, 'Related_Product__c'));
+  });
+
+  it('skips excluded lookup fields as filtered IN-clause pivots', () => {
+    const { childTask } = createInClauseFixture();
+    childTask.scriptObject.excludedFromInClauseFields = ['Product__c', 'Related_Product__c'];
+
+    const queries = childTask.createFilteredQueries('forwards', false);
+
+    assert.equal(queries.length, 1);
+    assert.ok(hasFilteredInClause(queries, 'Parent_Offer__c'));
+    assert.ok(!hasFilteredInClause(queries, 'Product__c'));
+    assert.ok(!hasFilteredInClause(queries, 'Related_Product__c'));
+  });
+
+  it('subtracts excluded lookup fields from the IN-clause allow-list case-insensitively', () => {
+    const { childTask } = createInClauseFixture();
+    childTask.scriptObject.includedInClauseFields = ['parent_offer__c', 'PRODUCT__C'];
+    childTask.scriptObject.excludedFromInClauseFields = ['product__c'];
+
+    const queries = childTask.createFilteredQueries('forwards', false);
+
+    assert.equal(queries.length, 1);
+    assert.ok(hasFilteredInClause(queries, 'Parent_Offer__c'));
+    assert.ok(!hasFilteredInClause(queries, 'Product__c'));
+  });
+});
+
 describe('MigrationJobTask field capability warnings', () => {
   it('warns only for explicit update query fields that are not updateable', () => {
     const originalLogger = Common.logger;

test/modules/script/script-loader.test.ts

@@ -75,6 +75,8 @@ describe('ScriptLoader', () => {
             {
               name: 'Account',
               query: 'SELECT Id, Name FROM Account',
+              includedInClauseFields: ['Parent_Offer__c', 'Product__c'],
+              excludedFromInClauseFields: ['Related_Product__c'],
               fieldMapping: [
                 {
                   targetObject: 'Account',
@@ -108,6 +110,8 @@ describe('ScriptLoader', () => {
       assert.ok(script.objectSets[0].objects[0] instanceof ScriptObject);
       assert.ok(script.objectSets[0].objects[0].fieldMapping[0] instanceof ScriptMappingItem);
       assert.equal(script.objectSets[0].objects[0].name, '');
+      assert.deepEqual(script.objectSets[0].objects[0].includedInClauseFields, ['Parent_Offer__c', 'Product__c']);
+      assert.deepEqual(script.objectSets[0].objects[0].excludedFromInClauseFields, ['Related_Product__c']);
       assert.ok(script.orgs[0] instanceof ScriptOrg);
       assert.ok(script.workingJson);
     } finally {
@@ -264,6 +268,8 @@ describe('ScriptLoader', () => {
           master: null,
           beforeAddons: null,
           excludedFields: null,
+          includedInClauseFields: null,
+          excludedFromInClauseFields: null,
         },
       ],
     };
@@ -305,6 +311,8 @@ describe('ScriptLoader', () => {
       assert.equal(object.master, null);
       assert.equal(object.beforeAddons, null);
       assert.equal(object.excludedFields, null);
+      assert.equal(object.includedInClauseFields, null);
+      assert.equal(object.excludedFromInClauseFields, null);
     } finally {
       Common.csvReadFileDelimiter = previousReadDelimiter;
       Common.csvWriteFileDelimiter = previousWriteDelimiter;
@@ -344,6 +352,8 @@ describe('ScriptLoader', () => {
           deleteOldData: undefined,
           beforeAddons: undefined,
           excludedFields: undefined,
+          includedInClauseFields: undefined,
+          excludedFromInClauseFields: undefined,
         },
       ],
     } as unknown as ScriptPayloadType;
@@ -372,6 +382,8 @@ describe('ScriptLoader', () => {
     assert.equal(object.master, false);
     assert.deepEqual(object.beforeAddons, []);
     assert.deepEqual(object.excludedFields, []);
+    assert.deepEqual(object.includedInClauseFields, []);
+    assert.deepEqual(object.excludedFromInClauseFields, []);
   });
 
   it('prepends flat objects to objectSets index 0 when both are present', async () => {