fix: Fixed #569

Author: hknokh2

package.json

@@ -83,7 +83,7 @@
   "repository": "forcedotcom/SFDX-Data-Move-Utility",
   "addons": {
     "run": {
-      "version": "1.4.0"
+      "version": "1.5.0"
     }
   },
   "scripts": {

src/addons/components/sfdmu-run/sfdmuRunAddonRuntime.ts

@@ -32,6 +32,7 @@ import AddonModule from '../common/addonModule';
 import { ISfdmuRunCustomAddonRuntime } from '../../modules/sfdmu-run/custom-addons/package';
 import { SfdmuRunCustomAddonService } from './custom';
 import ISfdmuRunScript from './ISfdmuRunScript';
+import { BulkApiV2_0Engine } from '../../../modules/components/api_engines/bulkApiV2_0Engine';
 
 
 
@@ -206,7 +207,7 @@ export default class SfdmuRunAddonRuntime extends AddonRuntime implements ISfdmu
           break;
 
         case API_ENGINE.BULK_API_V2:
-          apiEngine = new BulkApiV1_0Engine({
+          apiEngine = new BulkApiV2_0Engine({
             logger: this.#logger,
             connectionData: this.#script.targetOrg.connectionData,
             sObjectName,
@@ -277,7 +278,7 @@ export default class SfdmuRunAddonRuntime extends AddonRuntime implements ISfdmu
   async transferContentVersions(module: AddonModule, sourceVersions: ContentVersion[], maxChunkSize?: number): Promise<ContentVersion[]> {
 
     let _self = this;
-    maxChunkSize = maxChunkSize || CONSTANTS.MAX_CONTENT_VERSION_PROCESSING_MEMORY_SIZE;
+    maxChunkSize = maxChunkSize || CONSTANTS.DEFAULT_MAX_CHUNK_SIZE;
 
     // All Files of url types to upload ///
     let urlUploadJobs = new Array<ContentVersion>();

src/addons/modules/sfdmu-run/ExportFiles/index.ts

@@ -42,13 +42,22 @@ interface IOnExecuteArguments {
 
 
   /**
-   *
-   * For safe uploading binary data, the data is splitted into multiple chunks
-   * to be uploaded sequentially.
+   * For optimized porocessing, files are grouped into multiple chunks and uploaded sequentially.
    * This parameter defines the maximum size of each chunk (in bytes).
-   * Default to 15M.
+   * A size of a single file can exceed this value, but should be up to maxFileSize;
+   * Default: 15Mb.
+   * Maximum: 37Mb.
    */
   maxChunkSize: number;
+
+  /**
+   * Limits the maximal allowed size of a single file (in bytes) (ContentDocument.ContentSize field)
+   * Commonly, the SF API is limiting the max size of the API request to 52428800 bytes.
+   * So, the maximal file size can't be greater then 37Mb. Larger files are ignored and not uploaded.
+   * Default: 37Mb.
+   * Maximum: 37Mb
+   */
+  maxFileSize: number;
 }
 
 interface IDataToImport {
@@ -87,7 +96,7 @@ export default class ExportFiles extends SfdmuRunAddonModule {
         break;
 
       default:
-        // TODO: Something to do whe is a regular sObject type
+        // TODO: Something to do when it's a regular sObject type
         break;
     }
 
@@ -111,6 +120,17 @@ export default class ExportFiles extends SfdmuRunAddonModule {
       return;
     }
 
+    // Default values
+    args.maxFileSize = args.maxFileSize || CONSTANTS.DEFAULT_MAX_FILE_SIZE;
+    if (args.maxFileSize > CONSTANTS.MAX_FILE_SIZE) {
+      args.maxFileSize = CONSTANTS.MAX_FILE_SIZE;
+    }
+
+    args.maxChunkSize = args.maxChunkSize || CONSTANTS.DEFAULT_MAX_CHUNK_SIZE;
+    if (args.maxChunkSize > CONSTANTS.MAX_CHUNK_SIZE) {
+      args.maxChunkSize = CONSTANTS.MAX_CHUNK_SIZE;
+    }
+
     // Get the relevant parent task
     let task = this.runtime.pluginJob.tasks.find(task => task.sObjectName == context.objectName);
 
@@ -177,9 +197,17 @@ export default class ExportFiles extends SfdmuRunAddonModule {
 
       await this.runtime.transferContentVersions(this, versionsToProcess, args.maxChunkSize);
 
+      const failedRecordsCount = versionsToProcess.filter(item => item.isError).length;
+
       this.runtime.logFormattedInfo(this, SFDMU_RUN_ADDON_MESSAGES.ExportFiles_ProcessedRecords,
         String(versionsToProcess.length),
-        String(versionsToProcess.filter(item => item.isError).length));
+        String(failedRecordsCount));
+
+      if (failedRecordsCount == versionsToProcess.length) {
+        // All content versions are failed => Nothing to process
+        exportedFilesMap.clear();
+        versionsToProcess = [];
+      }
     }
 
 
@@ -384,7 +412,7 @@ export default class ExportFiles extends SfdmuRunAddonModule {
           filteredByDocIdsByField,
           'ContentVersion',
           sourceFiles.docIds,
-          'IsLatest = true');
+          `(IsLatest = true) AND (ContentDocument.ContentSize <= ${args.maxFileSize})`);
         if (args.sourceWhere) {
           queries = queries.map(query => query.replace('WHERE', 'WHERE (' + args.sourceWhere + ') AND (') + ')')
         }

src/modules/components/common_components/common.ts

@@ -1171,9 +1171,7 @@ export class Common {
         if (parsedWhere) {
           tempQuery.where.left.openParen = 1;
           tempQuery.where.left.closeParen = 1;
-          tempQuery.where.right = <WhereClause>{
-            left: parsedWhere.where.left
-          };
+          tempQuery.where.right = parsedWhere.where;
           tempQuery.where.operator = "AND";
         }
         yield composeQuery(tempQuery);

src/modules/components/common_components/statics.ts

@@ -380,7 +380,12 @@ export const CONSTANTS = {
 
 
   // ------ AddOns -------------------- //
-  MAX_CONTENT_VERSION_PROCESSING_MEMORY_SIZE: 15000000,
+  DEFAULT_MAX_CHUNK_SIZE: 15728640,
+  MAX_CHUNK_SIZE: 38797312,
+
+  DEFAULT_MAX_FILE_SIZE: 38797312,
+  MAX_FILE_SIZE: 38797312,
+  
 
 
   // ------- Headers ------------------ //