Skip to content

Commit e0c953a

Browse files
author
hknokh2
committed
feat: add object-level api mode controls and threshold-inclusive bulk behavior
1 parent f353bea commit e0c953a

File tree

12 files changed

+353
-10
lines changed

12 files changed

+353
-10
lines changed

‎custom-addon-sdk/interfaces/ISfdmuRunCustomAddonScript.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ import type { DATA_CACHE_TYPES } from './common.js';
2222
export default interface ISfdmuRunCustomAddonScript {
2323
orgs?: ISfdmuRunCustomAddonScriptOrg[];
2424
objects?: ISfdmuRunCustomAddonScriptObject[];
25+
objectSets?: Array<{ objects: ISfdmuRunCustomAddonScriptObject[] }>;
26+
objectsMap?: Map<string, ISfdmuRunCustomAddonScriptObject>;
27+
excludedObjects?: string[];
2528
job?: ISFdmuRunCustomAddonJob;
2629

2730
/**
@@ -53,6 +56,7 @@ export default interface ISfdmuRunCustomAddonScript {
5356
promptOnIssuesInCSVFiles?: boolean;
5457
validateCSVFilesOnly?: boolean;
5558
apiVersion?: string;
59+
groupQuery?: string;
5660
createTargetCSVFiles?: boolean;
5761
importCSVFilesAsIs?: boolean;
5862
alwaysUseRestApiToUpdateRecords?: boolean;
@@ -111,6 +115,7 @@ export default interface ISfdmuRunCustomAddonScript {
111115
*/
112116
objectSetIndex?: number;
113117
proxyUrl?: string;
118+
canModify?: string;
114119
binaryDataCache?: DATA_CACHE_TYPES;
115120
sourceRecordsCache?: DATA_CACHE_TYPES;
116121
parallelBinaryDownloads?: number;
@@ -120,6 +125,7 @@ export default interface ISfdmuRunCustomAddonScript {
120125
beforeAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[];
121126
afterAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[];
122127
dataRetrievedAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[];
128+
sourceTargetFieldMapping?: Map<string, unknown>;
123129

124130
/**
125131
* Returns all configured script objects.

‎custom-addon-sdk/interfaces/ISfdmuRunCustomAddonScriptObject.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,21 @@ export default interface ISfdmuRunCustomAddonScriptObject {
2525
deleteQuery?: string;
2626
operation?: OPERATION;
2727
externalId?: string;
28+
originalExternalId?: string;
29+
originalExternalIdIsEmpty?: boolean;
2830
deleteOldData?: boolean;
2931
deleteFromSource?: boolean;
3032
deleteByHierarchy?: boolean;
3133
hardDelete?: boolean;
34+
respectOrderByOnDeleteRecords?: boolean;
3235
updateWithMockData?: boolean;
3336
// mockCSVData?: boolean;
3437
sourceRecordsFilter?: string;
3538
targetRecordsFilter?: string;
3639
excluded?: boolean;
40+
useQueryAll?: boolean;
41+
queryAllTarget?: boolean;
42+
skipExistingRecords?: boolean;
3743
useCSVValuesMapping?: boolean;
3844
useFieldMapping?: boolean;
3945
/**
@@ -44,8 +50,12 @@ export default interface ISfdmuRunCustomAddonScriptObject {
4450
master?: boolean;
4551
excludedFields?: string[];
4652
excludedFromUpdateFields?: string[];
53+
excludedFieldsFromUpdate?: string[];
4754
restApiBatchSize?: number;
4855
bulkApiV1BatchSize?: number;
56+
alwaysUseBulkApiToUpdateRecords?: boolean;
57+
alwaysUseRestApi?: boolean;
58+
alwaysUseBulkApi?: boolean;
4959
parallelBulkJobs?: number;
5060
parallelRestJobs?: number;
5161

@@ -71,6 +81,21 @@ export default interface ISfdmuRunCustomAddonScriptObject {
7181
/** Extra fields to be updated on target records. */
7282
extraFieldsToUpdate: string[];
7383

84+
processAllSource?: boolean;
85+
processAllTarget?: boolean;
86+
isFromOriginalScript?: boolean;
87+
sourceSObjectDescribe?: unknown;
88+
targetSObjectDescribe?: unknown;
89+
isExtraObject?: boolean;
90+
91+
/**
92+
* Polymorphic lookup object hints defined in export.json.
93+
*/
94+
polymorphicLookups?: Array<{
95+
fieldName: string;
96+
referencedObjectType?: string;
97+
}>;
98+
7499
/**
75100
* True when the object is auto-added by the runtime.
76101
*/

‎custom-addon-sdk/interfaces/ISfdmuRunCustomAddonScriptOrg.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
* For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
66
*/
77

8+
import type { DATA_MEDIA_TYPE } from './common.js';
9+
810
/**
911
* The authentication data provided with the currently running {@link ISfdmuRunCustomAddonScript}.
1012
*
@@ -16,8 +18,28 @@
1618
export default interface ISfdmuRunCustomAddonScriptOrg {
1719
name: string;
1820
orgUserName: string;
21+
orgId: string;
1922
instanceUrl: string;
2023
accessToken: string;
24+
media?: DATA_MEDIA_TYPE;
25+
isSource?: boolean;
26+
orgDescribe?: Map<string, unknown>;
27+
isPersonAccountEnabled?: boolean;
28+
organizationType?: string;
29+
isSandbox?: boolean;
30+
isScratch?: boolean;
31+
connectionData?: {
32+
instanceUrl: string;
33+
accessToken: string;
34+
apiVersion: string;
35+
proxyUrl: string;
36+
};
37+
isConnected?: boolean;
38+
isDescribed?: boolean;
39+
objectNamesList?: string[];
40+
isProduction?: boolean;
41+
isDeveloper?: boolean;
42+
instanceDomain?: string;
2143

2244
/**
2345
* True when this org represents file-based media.

‎schemas/export.schema.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,18 @@
259259
"hardDelete": {
260260
"type": "boolean"
261261
},
262+
"respectOrderByOnDeleteRecords": {
263+
"type": "boolean"
264+
},
265+
"alwaysUseBulkApiToUpdateRecords": {
266+
"type": "boolean"
267+
},
268+
"alwaysUseRestApi": {
269+
"type": "boolean"
270+
},
271+
"alwaysUseBulkApi": {
272+
"type": "boolean"
273+
},
262274
"sourceRecordsFilter": {
263275
"type": "string"
264276
},

‎src/modules/api/ApiEngineFactory.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ export default class ApiEngineFactory {
8787
return API_ENGINE.BULK_API_V1;
8888
}
8989

90-
const bulkAllowed = amountToProcess > bulkThreshold && !alwaysUseRest && bulkSupported;
90+
const bulkAllowed = amountToProcess >= bulkThreshold && !alwaysUseRest && bulkSupported;
9191

9292
if (!bulkAllowed) {
9393
return API_ENGINE.REST_API;

‎src/modules/models/job/MigrationJobTask.ts‎

Lines changed: 79 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ type ProcessedFieldsToRemoveType = {
8484
notUpdateableFields: string[];
8585
};
8686

87+
type DmlExecutionSettingsType = {
88+
alwaysUseRest: boolean;
89+
bulkThreshold: number;
90+
restApiBatchSizeOverride?: number;
91+
};
92+
8793
const { parseQuery, composeQuery } = CjsDependencyAdapters.getSoqlParser();
8894

8995
/**
@@ -3471,14 +3477,16 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
34713477
return [];
34723478
}
34733479

3480+
const dmlSettings = this._resolveDmlExecutionSettings(operation);
3481+
const effectiveScriptObject = this._createEffectiveScriptObject(dmlSettings.restApiBatchSizeOverride);
34743482
const connection = await this._getConnectionAsync(org);
34753483
const engine = ApiEngineFactory.createEngine({
34763484
connection,
34773485
sObjectName,
34783486
amountToProcess: records.length,
3479-
bulkThreshold: this.job.script.bulkThreshold,
3480-
alwaysUseRest: this.job.script.alwaysUseRestApiToUpdateRecords,
3481-
forceBulk: operation === OPERATION.HardDelete,
3487+
bulkThreshold: dmlSettings.bulkThreshold,
3488+
alwaysUseRest: dmlSettings.alwaysUseRest,
3489+
forceBulk: operation === OPERATION.HardDelete && !dmlSettings.alwaysUseRest,
34823490
bulkApiVersion: this.job.script.bulkApiVersion,
34833491
});
34843492
const apiVersion = this._resolveApiVersion(connection);
@@ -3497,7 +3505,9 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
34973505
operation
34983506
)} engine=${engine.getEngineName()} apiVersion=${apiVersion} count=${records.length} allOrNone=${String(
34993507
this.job.script.allOrNone
3500-
)} updateRecordId=${String(updateRecordId)} finalAttempt=${String(isFinalDmlAttempt)}`
3508+
)} updateRecordId=${String(updateRecordId)} finalAttempt=${String(isFinalDmlAttempt)} alwaysUseRest=${String(
3509+
dmlSettings.alwaysUseRest
3510+
)} bulkThreshold=${dmlSettings.bulkThreshold} restBatchOverride=${dmlSettings.restApiBatchSizeOverride ?? 'none'}`
35013511
);
35023512

35033513
const executor = new ApiEngineExecutor({
@@ -3507,7 +3517,7 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
35073517
updateRecordId,
35083518
logger: this._getLogger(),
35093519
script: this.job.script,
3510-
scriptObject: this.scriptObject,
3520+
scriptObject: effectiveScriptObject,
35113521
isFinalDmlAttempt,
35123522
});
35133523
let processed: Array<Record<string, unknown>>;
@@ -3522,7 +3532,7 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
35223532
connection,
35233533
sObjectName,
35243534
amountToProcess: records.length,
3525-
bulkThreshold: this.job.script.bulkThreshold,
3535+
bulkThreshold: dmlSettings.bulkThreshold,
35263536
alwaysUseRest: true,
35273537
forceBulk: false,
35283538
bulkApiVersion: this.job.script.bulkApiVersion,
@@ -3554,7 +3564,7 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
35543564
updateRecordId,
35553565
logger: this._getLogger(),
35563566
script: this.job.script,
3557-
scriptObject: this.scriptObject,
3567+
scriptObject: effectiveScriptObject,
35583568
isFinalDmlAttempt,
35593569
});
35603570
processed = await restExecutor.executeCrudAsync();
@@ -3569,6 +3579,68 @@ export default class MigrationJobTask implements ISFdmuRunCustomAddonTask {
35693579
return processed;
35703580
}
35713581

3582+
/**
3583+
* Resolves effective DML engine settings for the current object and operation.
3584+
*
3585+
* @param operation - Operation to execute.
3586+
* @returns Effective DML settings.
3587+
*/
3588+
private _resolveDmlExecutionSettings(operation: OPERATION): DmlExecutionSettingsType {
3589+
const script = this.job.script;
3590+
const object = this.scriptObject;
3591+
const isDeleteOperation = this._isDeleteLikeOperation(operation);
3592+
let alwaysUseRest = script.alwaysUseRestApiToUpdateRecords || object.alwaysUseRestApi;
3593+
let bulkThreshold = script.bulkThreshold;
3594+
let restApiBatchSizeOverride: number | undefined;
3595+
3596+
if (object.respectOrderByOnDeleteRecords && isDeleteOperation) {
3597+
alwaysUseRest = true;
3598+
restApiBatchSizeOverride = 1;
3599+
}
3600+
3601+
if (!alwaysUseRest && (object.alwaysUseBulkApi || object.alwaysUseBulkApiToUpdateRecords)) {
3602+
bulkThreshold = 0;
3603+
}
3604+
3605+
return {
3606+
alwaysUseRest,
3607+
bulkThreshold,
3608+
restApiBatchSizeOverride,
3609+
};
3610+
}
3611+
3612+
/**
3613+
* Returns true when the operation is delete-like.
3614+
*
3615+
* @param operation - Operation to inspect.
3616+
* @returns True when delete-like.
3617+
*/
3618+
private _isDeleteLikeOperation(operation: OPERATION): boolean {
3619+
void this;
3620+
return (
3621+
operation === OPERATION.Delete ||
3622+
operation === OPERATION.DeleteHierarchy ||
3623+
operation === OPERATION.DeleteSource ||
3624+
operation === OPERATION.HardDelete
3625+
);
3626+
}
3627+
3628+
/**
3629+
* Creates a script-object view with an optional REST batch-size override.
3630+
*
3631+
* @param restApiBatchSizeOverride - Optional REST batch-size override.
3632+
* @returns Effective script object for API executor options.
3633+
*/
3634+
private _createEffectiveScriptObject(restApiBatchSizeOverride?: number): ScriptObject {
3635+
if (!restApiBatchSizeOverride || this.scriptObject.restApiBatchSize === restApiBatchSizeOverride) {
3636+
return this.scriptObject;
3637+
}
3638+
3639+
const clone = Object.assign(new ScriptObject(), this.scriptObject);
3640+
clone.restApiBatchSize = restApiBatchSizeOverride;
3641+
return clone;
3642+
}
3643+
35723644
/**
35733645
* Returns true when hard delete should retry via REST API.
35743646
*

‎src/modules/models/job/TaskOrgData.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,19 @@ export default class TaskOrgData {
137137
* @returns True when Bulk API query should be used.
138138
*/
139139
public get useBulkQueryApi(): boolean {
140-
const threshold = this.task.job.script.queryBulkApiThreshold;
141-
return Boolean(threshold && this.totalRecordCount >= threshold);
140+
const object = this.task.scriptObject;
141+
if (object.alwaysUseRestApi) {
142+
return false;
143+
}
144+
if (object.alwaysUseBulkApi) {
145+
return true;
146+
}
147+
148+
const threshold = Number(this.task.job.script.queryBulkApiThreshold);
149+
if (!Number.isFinite(threshold)) {
150+
return false;
151+
}
152+
return this.totalRecordCount >= threshold;
142153
}
143154

144155
// ------------------------------------------------------//

‎src/modules/models/script/ScriptObject.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,27 @@ export default class ScriptObject {
232232
*/
233233
public hardDelete = false;
234234

235+
/**
236+
* Enforces REST delete one-by-one mode to preserve ORDER BY intent.
237+
* Applies only to delete-like operations.
238+
*/
239+
public respectOrderByOnDeleteRecords = false;
240+
241+
/**
242+
* Forces Bulk API for object DML operations when REST override is not active.
243+
*/
244+
public alwaysUseBulkApiToUpdateRecords = false;
245+
246+
/**
247+
* Forces REST API usage for object queries and object DML operations.
248+
*/
249+
public alwaysUseRestApi = false;
250+
251+
/**
252+
* Forces Bulk API usage for object queries and object DML operations.
253+
*/
254+
public alwaysUseBulkApi = false;
255+
235256
/**
236257
* Source records filter expression applied before DML.
237258
*/

‎test/modules/api/api-engine-factory.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,20 @@ describe('ApiEngineFactory', () => {
8787
assert.equal(engine.getEngineType(), API_ENGINE.BULK_API_V2);
8888
});
8989

90+
it('uses Bulk API when amount equals bulk threshold', () => {
91+
const connection = { bulk2: {} } as Connection;
92+
const engine = ApiEngineFactory.createEngine({
93+
connection,
94+
sObjectName: 'Account',
95+
amountToProcess: 100,
96+
bulkThreshold: 100,
97+
alwaysUseRest: false,
98+
bulkApiVersion: '2.0',
99+
});
100+
101+
assert.equal(engine.getEngineType(), API_ENGINE.BULK_API_V2);
102+
});
103+
90104
it('keeps REST API for unsupported objects even when forceBulk is true', () => {
91105
const connection = { bulk2: {} } as Connection;
92106
const unsupportedObject = NOT_SUPPORTED_OBJECTS_IN_BULK_API[0] ?? 'Attachment';

0 commit comments

Comments
 (0)