Merge pull request #567 from hknokh/master

fix: Separated source/target dirs for ObjectSets

Author: hknokh

src/modules/components/common_components/statics.ts

@@ -51,9 +51,11 @@ export const CONSTANTS = {
   ADDON_TEMP_RELATIVE_FOLDER: 'temp_%s/',
 
   CSV_SOURCE_SUB_DIRECTORY: "source",
+  RAW_SOURCE_SUB_DIRECTORY: "objectset_source",
   CSV_TARGET_SUB_DIRECTORY: "target",
   OBJECT_SET_SUBDIRECTORY_PREFIX : '/object-set-',
   BINARY_CACHE_SUB_DIRECTORY: "binary_cache",
+  REPORTS_SUB_DIRECTORY: "reports",
   SOURCE_RECORDS_CACHE_SUB_DIRECTORY: "source_records_cache",
   CSV_SOURCE_FILE_SUFFIX: "_source",
   CSV_TARGET_FILE_SUFFIX: "_target",

src/modules/models/common_models/helpers_classes.ts

@@ -11,27 +11,27 @@ import 'es6-shim';
 import { Type } from 'class-transformer';
 
 import {
-  MigrationJobTask,
-  ScriptObject,
-  ScriptOrg,
-  SFieldDescribe,
+    MigrationJobTask,
+    ScriptObject,
+    ScriptOrg,
+    SFieldDescribe,
 } from '../';
 import {
-  ISfdmuRunCustomAddonProcessedData,
+    ISfdmuRunCustomAddonProcessedData,
 } from '../../../addons/modules/sfdmu-run/custom-addons/package';
 import { Common } from '../../components/common_components/common';
 import {
-  DATA_MEDIA_TYPE,
-  OPERATION,
+    DATA_MEDIA_TYPE,
+    OPERATION,
 } from '../../components/common_components/enumerations';
 import { RESOURCES } from '../../components/common_components/logger';
 import { CONSTANTS } from '../../components/common_components/statics';
 import { ICsvChunk } from '../api_models';
 import ScriptAddonManifestDefinition
-  from '../script_models/scriptAddonManifestDefinition';
+    from '../script_models/scriptAddonManifestDefinition';
 import {
-  IMissingParentLookupRecordCsvRow,
-  IOrgConnectionData,
+    IMissingParentLookupRecordCsvRow,
+    IOrgConnectionData,
 } from './helper_interfaces';
 
 type IProcessedData = ISfdmuRunCustomAddonProcessedData;
@@ -139,7 +139,7 @@ export class TaskData {
      * @memberof TaskData
      */
     get csvFilename(): string {
-        return this.task.getCSVFilename(this.task.script.basePath);
+        return this.task.getCSVFilename(this.task.script.rawSourceDirectory);
     }
 
     /**
@@ -150,8 +150,7 @@ export class TaskData {
      * @memberof TaskData
      */
     get sourceCsvFilename(): string {
-        return this.task.getCSVFilename(this.task.script.sourceDirectory,
-            CONSTANTS.CSV_SOURCE_FILE_SUFFIX);
+        return this.task.getCSVFilename(this.task.script.sourceDirectory, CONSTANTS.CSV_SOURCE_FILE_SUFFIX);
     }
 
     /**
@@ -165,8 +164,6 @@ export class TaskData {
      */
     getTargetCSVFilename(operation: OPERATION, fileNameSuffix?: string): string {
         return TaskData.getTargetCSVFilename(this.task.script.targetDirectory, this.task.sObjectName, operation, fileNameSuffix);
-        // return this.task.getCSVFilename(this.task.script.targetDirectory,
-        //     `_${ScriptObject.getStrOperation(operation).toLowerCase()}${fileNameSuffix || ""}${CONSTANTS.CSV_TARGET_FILE_SUFFIX}`);
     }
 
     /**

src/modules/models/job_models/migrationJob.ts

@@ -732,7 +732,7 @@ export default class MigrationJob {
    * @memberof MigrationJob
    */
   async saveCSVFileAsync(fileName: string, data: Array<any>, alwaysCreateFile: boolean = true): Promise<void> {
-    let filePath = path.join(this.script.basePath, fileName);
+    let filePath = path.join(this.script.reportsDirectory, fileName);
     this.logger.infoVerbose(RESOURCES.writingCsvFile, filePath);
     await Common.writeCsvFileAsync(filePath, data, alwaysCreateFile);
   }
@@ -1022,7 +1022,8 @@ export default class MigrationJob {
           // Report csv issues
           await self.saveCSVFileAsync(CONSTANTS.CSV_ISSUES_ERRORS_FILENAME, self.csvIssues);
         },
-        String(self.csvIssues.length), CONSTANTS.CSV_ISSUES_ERRORS_FILENAME);
+        String(self.csvIssues.length),
+        CONSTANTS.CSV_ISSUES_ERRORS_FILENAME);
     }
   }
 }

src/modules/models/script_models/script.ts

@@ -98,6 +98,7 @@ export default class Script implements IAppScript, ISfdmuRunScript {
   proxyUrl: string;
   csvReadFileDelimiter: ',' | ';' = ",";
   csvWriteFileDelimiter: ',' | ';' = ",";
+  useSeparatedCSVFiles: false;
 
   binaryDataCache: DATA_CACHE_TYPES = DATA_CACHE_TYPES.InMemory;
   sourceRecordsCache: DATA_CACHE_TYPES = DATA_CACHE_TYPES.InMemory;
@@ -160,7 +161,7 @@ export default class Script implements IAppScript, ISfdmuRunScript {
     return path.join(
       this.basePath,
       CONSTANTS.CSV_TARGET_SUB_DIRECTORY +
-      (!this.objectSetIndex ? '' : `${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex}`)
+      (!this.objectSetIndex ? '' : `${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex + 1}`)
     );
   }
 
@@ -175,7 +176,7 @@ export default class Script implements IAppScript, ISfdmuRunScript {
     return path.join(
       this.basePath,
       CONSTANTS.CSV_SOURCE_SUB_DIRECTORY +
-      (!this.objectSetIndex ? '' : `${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex}`)
+      (!this.objectSetIndex ? '' : `${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex + 1}`)
     );
   }
 
@@ -186,6 +187,34 @@ export default class Script implements IAppScript, ISfdmuRunScript {
     return this.sourceDirectoryPath;
   }
 
+  get rawSourceDirectory(): string {
+    if (!fs.existsSync(this.rawSourceDirectoryPath)) {
+      fs.mkdirSync(this.rawSourceDirectoryPath, { recursive: true });
+    }
+    return this.rawSourceDirectoryPath;
+  }
+
+  get rawSourceDirectoryPath(): string {
+    return path.join(
+      this.basePath,
+      (!this.objectSetIndex || !this.useSeparatedCSVFiles ? '' : `${CONSTANTS.RAW_SOURCE_SUB_DIRECTORY}/${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex + 1}`)
+    );
+  }
+
+  get reportsDirectoryPath(): string {
+    return path.join(
+      this.basePath,
+      (!this.objectSetIndex ? '' : `${CONSTANTS.REPORTS_SUB_DIRECTORY}/${CONSTANTS.OBJECT_SET_SUBDIRECTORY_PREFIX}${this.objectSetIndex + 1}`)
+    );
+  }
+
+  get reportsDirectory(): string {
+    if (!fs.existsSync(this.reportsDirectoryPath)) {
+      fs.mkdirSync(this.reportsDirectoryPath, { recursive: true });
+    }
+    return this.reportsDirectoryPath;
+  }
+
   get binaryCacheDirectoryPath(): string {
     return path.join(this.basePath, CONSTANTS.BINARY_CACHE_SUB_DIRECTORY, this.sourceOrg.orgUserName);
   }