Refactor account migration and export logic to support columnar format and improve chunk handling

Author: marioserrano09

extensions/saas/sources/migration/ARCHITECTURE.md

@@ -108,27 +108,29 @@ OutputStream (raw or GZIPOutputStream)
             ├── Write "account": AccountDTO object
             └── Write "entities": {
                     For each entityClass in topological order:
-                        Write entityClass.getName(): [
-                            LOOP (pagination by chunks):
-                                count = CrudService.count(entityClass, {accountId})
-                                for page 1..N:
-                                    chunk = CrudService.find(entityClass, {accountId, paginator})
-                                    for each record:
-                                        write as JSON object (flat map, refs as _ref_id)
-                        ]
+                        Write entityClass.getName(): {
+                            "fields": ["id", "name", "category_ref_id", ...],
+                            "rows": [
+                                [1, "John", 3],
+                                [2, "Cindy", null],
+                                ...
+                            ]
+                        }
                 }
 ```
 
-### Serialization of a Single Entity
+### Serialization of a Single Entity (Columnar Format)
 
+Columns are built once per entity type from `EntityType.getSingularAttributes()`:
 ```
-For each SingularAttribute in EntityType:
-  BASIC / EMBEDDED  → write field value directly
-  MANY_TO_ONE / ONE_TO_ONE → write {fieldName}_ref_id: <referenced entity's id>
-  ONE_TO_MANY / MANY_TO_MANY → SKIP (reconstructed during import via child entities)
+id → always first column
+For each SingularAttribute (excluding id):
+  BASIC / EMBEDDED  → column name = fieldName, value written directly
+  MANY_TO_ONE / ONE_TO_ONE → column name = fieldName + "_ref_id", value = referenced PK
+  ONE_TO_MANY / MANY_TO_MANY / ELEMENT_COLLECTION → SKIP
 ```
 
-Fields annotated with `@ExportIgnore` are skipped.
+Fields annotated with `@ExportIgnore` are skipped. Missing or inaccessible fields write `null`.
 
 ---
 
@@ -144,7 +146,9 @@ InputStream (auto-detected: raw or GZIPInputStream)
             └── Read "entities" → {
                     For each entityClassName:
                         resolve class → Class.forName(entityClassName)
-                        For each JSON record in array (chunked):
+                        Read "fields": [...] → ordered column names
+                        For each row array (chunked):
+                            reconstruct JsonNode from fields + row values
                             deserialize → entity instance
                             resolve _ref_id references via idMappings
                             set accountId = targetAccountId
@@ -204,7 +208,7 @@ saas_export_42_20260614T100500.json[.gz]
 
 ```json
 {
-  "version": "1",
+  "version": "2",
   "exportedAt": "2026-06-14T10:05:00",
   "sourceAccountId": 42,
   "identityStrategy": "KEEP_IDS",
@@ -216,23 +220,34 @@ saas_export_42_20260614T100500.json[.gz]
     ...
   },
   "entities": {
-    "tools.dynamia.modules.saas.jpa.AccountParameter": [
-      { "id": 1, "accountId": 42, "name": "theme", "value": "dark" }
-    ],
-    "com.example.Customer": [
-      { "id": 10, "accountId": 42, "name": "John", "category_ref_id": 3 }
-    ],
-    "com.example.Order": [
-      { "id": 100, "accountId": 42, "customer_ref_id": 10, "total": 99.99 }
-    ]
+    "tools.dynamia.modules.saas.jpa.AccountParameter": {
+      "fields": ["id", "accountId", "name", "value"],
+      "rows": [
+        [1, 42, "theme", "dark"]
+      ]
+    },
+    "com.example.Customer": {
+      "fields": ["id", "accountId", "name", "category_ref_id"],
+      "rows": [
+        [10, 42, "John", 3]
+      ]
+    },
+    "com.example.Order": {
+      "fields": ["id", "accountId", "total", "customer_ref_id"],
+      "rows": [
+        [100, 42, 99.99, 10]
+      ]
+    }
   }
 }
 ```
 
 **Key conventions:**
+- Each entity section is an object with `fields` (column names) and `rows` (value arrays).
 - `{fieldName}_ref_id` encodes a `@ManyToOne` / `@OneToOne` reference by its primary key.
 - The `account` section makes the package self-describing.
 - Entities appear in topological order (parents before children).
+- Format version `"2"` uses the columnar layout; version `"1"` (legacy) used per-row objects.
 
 ---
 

extensions/saas/sources/migration/pom.xml

@@ -27,7 +27,7 @@
         <version>26.6.0</version>
     </parent>
 
-    <name>DynamiaModules - SaaS Tenant Mobility (Migration)</name>
+    <name>DynamiaModules - SaaS Migration</name>
     <artifactId>tools.dynamia.modules.saas.migration</artifactId>
     <url>https://www.dynamia.tools/modules/saas/migration</url>
     <description>

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/api/AccountExportOptions.java

@@ -19,10 +19,12 @@
  */
 public class AccountExportOptions {
 
+    public static final int DEFAULT_CHUNK_SIZE = 5000;
+
     /**
-     * Number of records to read from DB per pagination page. Default: 500.
+     * Number of records to read from DB per pagination page. Default: 5000.
      */
-    private int chunkSize = 500;
+    private int chunkSize = DEFAULT_CHUNK_SIZE;
 
     /**
      * When {@code true}, the output stream is wrapped in GZIP compression.
@@ -75,7 +77,7 @@ public AccountExportOptions label(String label) {
 
     public int getChunkSize() {
         if (chunkSize <= 0) {
-            chunkSize = 500;
+            chunkSize = DEFAULT_CHUNK_SIZE;
         }
         return chunkSize;
     }

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/api/AccountMigrationJobDto.java

@@ -10,10 +10,12 @@
  */
 package tools.dynamia.modules.saas.migration.api;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import tools.dynamia.modules.saas.migration.domain.AccountJobStatus;
 import tools.dynamia.modules.saas.migration.domain.AccountJobType;
 import tools.dynamia.modules.saas.migration.domain.AccountMigrationJob;
 
+import java.time.Duration;
 import java.time.LocalDateTime;
 
 /**
@@ -22,6 +24,7 @@
  *
  * @author Mario Serrano Leones
  */
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public record AccountMigrationJobDto(
         Long id,
         String uuid,
@@ -31,18 +34,28 @@ public record AccountMigrationJobDto(
         AccountJobStatus status,
         int progress,
         String progressMessage,
+        long records,
         String errorMessage,
         String downloadUrl,
         LocalDateTime createdAt,
         LocalDateTime startedAt,
         LocalDateTime finishedAt
 ) {
 
-    /** Convenience: returns {@code true} when the job has reached a terminal state. */
+    /**
+     * Convenience: returns {@code true} when the job has reached a terminal state.
+     */
     public boolean isFinished() {
         return status == AccountJobStatus.COMPLETED
                 || status == AccountJobStatus.FAILED
                 || status == AccountJobStatus.CANCELLED;
     }
+
+    public Duration getDuration() {
+        if (startedAt != null && finishedAt != null) {
+            return Duration.between(startedAt, finishedAt);
+        }
+        return null;
+    }
 }
 

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/api/MigrationProgress.java

@@ -13,25 +13,28 @@
 /**
  * Carries progress information during a tenant migration operation.
  *
- * @param processedRecords Total records processed so far.
- * @param totalRecords     Total records expected (0 if unknown).
- * @param message          Human-readable description of the current step.
+ * @param processedEntities Total entities processed so far.
+ * @param totalEntities     Total entities expected (0 if unknown).
+ * @param message           Human-readable description of the current step.
+ * @param processedRecords  Total records processed so far (across all entities).
  * @author Mario Serrano Leones
  */
-public record MigrationProgress(long processedRecords, long totalRecords, String message) {
+public record MigrationProgress(long processedEntities, long totalEntities, String message, long processedRecords) {
 
-    /** Returns the progress as a percentage (0–100), or -1 if total is unknown. */
+    /**
+     * Returns the progress as a percentage (0–100), or -1 if total is unknown.
+     */
     public int percentage() {
-        if (totalRecords <= 0) return -1;
-        return (int) Math.min(100, (processedRecords * 100L) / totalRecords);
+        if (totalEntities <= 0) return -1;
+        return (int) Math.min(100, (processedEntities * 100L) / totalEntities);
     }
 
     @Override
     public String toString() {
-        if (totalRecords > 0) {
-            return "[%d%%] %s (%d / %d)".formatted(percentage(), message, processedRecords, totalRecords);
+        if (totalEntities > 0) {
+            return "[%d%%] %s (%d / %d)".formatted(percentage(), message, processedEntities, totalEntities);
         }
-        return "[?] %s (%d processed)".formatted(message, processedRecords);
+        return "[?] %s (%d processed)".formatted(message, processedEntities);
     }
 }
 

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/discovery/AccountEntityDiscovery.java

@@ -15,6 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import tools.dynamia.integration.sterotypes.Service;
+import tools.dynamia.modules.entityfile.domain.EntityFile;
 import tools.dynamia.modules.saas.api.AccountAware;
 import tools.dynamia.modules.saas.api.AccountExportIgnore;
 import tools.dynamia.modules.saas.domain.Account;
@@ -60,6 +61,8 @@ public List<Class<?>> discoverExportableEntities() {
 
         // Always include Account as the tenant root
         exportable.add(Account.class);
+
+
         log.debug("[Migration] Always including: {}", Account.class.getName());
 
         for (EntityType<?> entityType : managedTypes) {
@@ -84,6 +87,7 @@ public List<Class<?>> discoverExportableEntities() {
             exportable.add(javaType);
             log.debug("[Migration] Discovered exportable entity: {}", javaType.getName());
         }
+        exportable.add(EntityFile.class); // EntityFile is used for storing exported file metadata and must be included even if not AccountAware
 
         log.info("[Migration] Discovered {} exportable entity types", exportable.size());
         return exportable;

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/domain/AccountMigrationJob.java

@@ -72,6 +72,7 @@ public class AccountMigrationJob extends SimpleEntity {
      * Completion percentage 0–100.
      */
     private int progress;
+    private long records;
 
     @Column(length = 2000)
     private String progressMessage;
@@ -140,9 +141,10 @@ public void markCancelled(String reason) {
     /**
      * Update running progress (0-100) and an optional human-readable message.
      */
-    public void updateProgress(int progress, String message) {
+    public void updateProgress(int progress, String message, long records) {
         this.progress = Math.min(100, Math.max(0, progress));
         this.progressMessage = StringUtils.truncate(message, 1999);
+        this.records = records;
     }
 
     /**
@@ -264,5 +266,13 @@ public void setOptionsJson(String optionsJson) {
     public String toString() {
         return "TenantMobilityJob{uuid=" + uuid + ", type=" + jobType + ", status=" + status + "}";
     }
+
+    public long getRecords() {
+        return records;
+    }
+
+    public void setRecords(long records) {
+        this.records = records;
+    }
 }
 

extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/pipeline/ExportConstants.java

@@ -21,7 +21,7 @@ private ExportConstants() {
     }
 
     /** Current format version written to every export file. */
-    public static final String FORMAT_VERSION = "1";
+    public static final String FORMAT_VERSION = "2";
 
     /**
      * Suffix appended to field names when serializing {@code @ManyToOne} /
@@ -48,5 +48,11 @@ private ExportConstants() {
 
     /** Top-level JSON field for the identity strategy name. */
     public static final String FIELD_IDENTITY_STRATEGY = "identityStrategy";
+
+    /** Per-entity field listing the ordered column names in the columnar format. */
+    public static final String FIELD_FIELDS = "fields";
+
+    /** Per-entity field containing the data rows as value arrays in the columnar format. */
+    public static final String FIELD_ROWS = "rows";
 }