Skip to content

Commit 458cd44

Browse files
marioserrano09marioserrano09
committed
Enhance MigrationProgress to support partial updates and refactor progress reporting in migration jobs
1 parent 534c5c5 commit 458cd44

6 files changed

Lines changed: 80 additions & 41 deletions

File tree

‎extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/api/MigrationProgress.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,22 @@
1919
* @param processedRecords Total records processed so far (across all entities).
2020
* @author Mario Serrano Leones
2121
*/
22-
public record MigrationProgress(long processedEntities, long totalEntities, String message, long processedRecords) {
22+
public record MigrationProgress(long processedEntities, long totalEntities, String message, long processedRecords,
23+
boolean partial) {
24+
25+
public static MigrationProgress partial(long processed) {
26+
return new MigrationProgress(0, 0, null, processed, true);
27+
}
28+
29+
public static MigrationProgress of(long processedEntities, long totalEntities, String message, long processedRecords) {
30+
return new MigrationProgress(processedEntities, totalEntities, message, processedRecords, false);
31+
}
2332

2433
/**
2534
* Returns the progress as a percentage (0–100), or -1 if total is unknown.
2635
*/
2736
public int percentage() {
28-
if (totalEntities <= 0) return -1;
37+
if (totalEntities <= 0) return 0;
2938
return (int) Math.min(100, (processedEntities * 100L) / totalEntities);
3039
}
3140

‎extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/domain/AccountMigrationJob.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@
1111
package tools.dynamia.modules.saas.migration.domain;
1212

1313
import jakarta.persistence.*;
14+
import org.hibernate.annotations.DynamicUpdate;
1415
import tools.dynamia.commons.StringUtils;
1516
import tools.dynamia.domain.jpa.SimpleEntity;
1617
import tools.dynamia.modules.entityfile.domain.EntityFile;
1718
import tools.dynamia.modules.saas.migration.api.AccountExportOptions;
1819
import tools.dynamia.modules.saas.migration.api.AccountImportOptions;
20+
import tools.dynamia.modules.saas.migration.api.MigrationProgress;
1921

2022
import java.io.Serializable;
2123
import java.time.LocalDateTime;
@@ -31,7 +33,8 @@
3133
* @author Mario Serrano Leones
3234
*/
3335
@Entity
34-
@Table(name = "saas_migration_jobs")
36+
@Table(name = "saas_migration_jobs", indexes = @Index(name = "idx_saas_migration_jobs_uuid", columnList = "uuid"))
37+
@DynamicUpdate
3538
public class AccountMigrationJob extends SimpleEntity {
3639

3740
// ─── Identity ──────────────────────────────────────────────────────────────
@@ -140,10 +143,16 @@ public void markCancelled(String reason) {
140143
/**
141144
* Update running progress (0-100) and an optional human-readable message.
142145
*/
143-
public void updateProgress(int progress, String message, long records) {
144-
this.progress = Math.min(100, Math.max(0, progress));
145-
this.progressMessage = StringUtils.truncate(message, 1999);
146-
this.records = records;
146+
public void updateProgress(MigrationProgress progress) {
147+
148+
if (progress.partial()) {
149+
this.records += progress.processedRecords();
150+
} else {
151+
this.progress = progress.percentage();
152+
this.progressMessage = StringUtils.truncate(progress.message(), 1999);
153+
this.records = progress.processedRecords();
154+
}
155+
147156
}
148157

149158
/**

‎extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/pipeline/ExportPipeline.java‎

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.hibernate.Session;
2020
import org.springframework.beans.factory.annotation.Qualifier;
2121
import org.springframework.util.ReflectionUtils;
22+
import tools.dynamia.commons.StopWatch;
2223
import tools.dynamia.commons.logger.LoggingService;
2324
import tools.dynamia.domain.jpa.JpaUtils;
2425
import tools.dynamia.domain.services.CrudService;
@@ -280,15 +281,13 @@ private void exportEntitiesInParallel(Path tempDir,
280281
semaphore.acquire();
281282
try {
282283
if (token != null && token.isCancelled()) return 0L;
283-
long count = exportEntityToFile(tempDir, entityClass, accountId, options, token);
284+
long count = exportEntityToFile(tempDir, entityClass, accountId, options, token, listener);
284285

285286
// Report progress immediately upon completion, not waiting for topological order
286287
long processed = processedTypes.incrementAndGet();
287288
long records = totalRecords.addAndGet(count);
288289
if (listener != null) {
289-
listener.onProgress(new MigrationProgress(processed, totalTypes,
290-
"Exported " + entityClass.getSimpleName() + " (" + count + " records)",
291-
records));
290+
listener.onProgress(MigrationProgress.of(processed, totalTypes, "Exported " + entityClass.getSimpleName(), records));
292291
}
293292
return count;
294293
} finally {
@@ -334,7 +333,8 @@ private void exportEntitiesInParallel(Path tempDir,
334333
*/
335334
private long exportEntityToFile(Path tempDir, Class<?> entityClass,
336335
Serializable accountId, AccountExportOptions options,
337-
CancellationToken token) throws IOException {
336+
CancellationToken token,
337+
MigrationProgressListener listener) throws IOException {
338338

339339
Path filePath = tempDir.resolve(entityFileName(accountId, entityClass));
340340
EntityManager localEm = emf.createEntityManager();
@@ -353,6 +353,8 @@ private long exportEntityToFile(Path tempDir, Class<?> entityClass,
353353
localEm.clear();
354354
localEm.close();
355355

356+
357+
logger.info("[Migration/Export] {} with {} columns", entityClass.getSimpleName(), columns.size());
356358
try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(filePath), ENTITY_BUFFER_SIZE);
357359
JsonGenerator gen = objectMapper.createGenerator(out)) {
358360

@@ -368,12 +370,14 @@ private long exportEntityToFile(Path tempDir, Class<?> entityClass,
368370

369371
gen.writeName(ExportConstants.FIELD_ROWS);
370372
gen.writeStartArray();
371-
long processed = writeEntityRows(gen, entityClass, accountId, options, token, columns);
373+
long startTime = System.currentTimeMillis();
374+
long processed = writeEntityRows(gen, entityClass, accountId, options, token, columns, listener);
375+
long endTime = System.currentTimeMillis();
372376
gen.writeEndArray();
373377

374378
gen.writeEndObject();
375379

376-
logger.info("[Migration/Export] {} → {} records written", entityClass.getSimpleName(), processed);
380+
logger.info("[Migration/Export] {} → {} records written in {}ms", entityClass.getSimpleName(), processed, (endTime - startTime));
377381
return processed;
378382
}
379383
} finally {
@@ -411,7 +415,8 @@ private long writeEntityRows(JsonGenerator gen,
411415
Serializable accountId,
412416
AccountExportOptions options,
413417
CancellationToken token,
414-
List<ColumnDef> columns) throws IOException {
418+
List<ColumnDef> columns,
419+
MigrationProgressListener listener) throws IOException {
415420
if (Account.class.equals(entityClass)) {
416421
return 0;
417422
}
@@ -424,10 +429,12 @@ private long writeEntityRows(JsonGenerator gen,
424429

425430

426431
do {
432+
long qStartTime = System.currentTimeMillis();
427433
List<Object> page = queryEntityDataPage(entityClass, accountId, lastId, simpleName, chunkSize);
428-
logger.info("[Migration/Export] {} - page {} with {} records. Rows processed {}", simpleName, pageNum, page.size(), processed);
434+
long qEndTime = System.currentTimeMillis();
429435
pageNum++;
430436

437+
long startTime = System.currentTimeMillis();
431438
for (Object entity : page) {
432439
if (token != null && token.isCancelled()) break;
433440
if (entity != null) {
@@ -439,6 +446,14 @@ private long writeEntityRows(JsonGenerator gen,
439446
}
440447
processed++;
441448
}
449+
long endTime = System.currentTimeMillis();
450+
451+
logger.info("[Migration/Export] {} - page {} with {} records. Query={}ms Write={}ms. Rows={} ",
452+
simpleName, pageNum, page.size(), (qEndTime - qStartTime), (endTime - startTime), processed);
453+
454+
if (listener != null) {
455+
listener.onProgress(MigrationProgress.partial(processed));
456+
}
442457
if (page.size() < chunkSize || (token != null && token.isCancelled())) break;
443458

444459
} while (true);
@@ -560,32 +575,30 @@ private List<ColumnDef> buildColumns(EntityType<?> entityType) {
560575

561576
private void writeEntityRow(JsonGenerator gen, Object entity, List<ColumnDef> columns)
562577
throws IOException {
563-
gen.writeStartArray();
564-
for (ColumnDef col : columns) {
578+
579+
580+
Object[] row = new Object[columns.size()];
581+
582+
for (int i = 0; i < columns.size(); i++) {
583+
ColumnDef col = columns.get(i);
584+
565585
try {
566586
Object value = col.field().get(entity);
567587

568588
if (col.type() == PersistentAttributeType.MANY_TO_ONE
569589
|| col.type() == PersistentAttributeType.ONE_TO_ONE) {
570-
if (value != null) {
571-
gen.writePOJO(JpaUtils.getJPAIdValue(value));
572-
} else {
573-
gen.writeNull();
574-
}
590+
row[i] = value != null
591+
? JpaUtils.getJPAIdValue(value)
592+
: null;
575593
} else {
576-
objectMapper.writeValue(gen, value);
594+
row[i] = value;
577595
}
578-
} catch (IllegalAccessException e) {
579-
logger.debug("[Migration/Export] Cannot access field {} on {}: {}",
580-
col.field().getName(), entity.getClass().getSimpleName(), e.getMessage());
581-
gen.writeNull();
582596
} catch (Exception e) {
583-
logger.debug("[Migration/Export] Skipping field {} due to: {}",
584-
col.field().getName(), e.getMessage());
585-
gen.writeNull();
597+
row[i] = null;
586598
}
587599
}
588-
gen.writeEndArray();
600+
601+
gen.writePOJO(row);
589602
}
590603

591604
// ─────────────────────────────────────────────────────────────────────────

‎extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/pipeline/ImportPipeline.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,14 @@ private void importFromZip(InputStream source,
168168
}
169169

170170
if (listener != null) {
171-
listener.onProgress(new MigrationProgress(total, total, "Import complete", total));
171+
listener.onProgress(MigrationProgress.of(total, total, "Import complete", total));
172172
}
173173
log.info("[Migration/Import] ZIP import complete — {} total records", total);
174174
}
175175

176-
/** Reads and logs metadata from manifest.json without loading it into memory. */
176+
/**
177+
* Reads and logs metadata from manifest.json without loading it into memory.
178+
*/
177179
private void logManifestInfo(ZipInputStream zipIn) {
178180
try {
179181
JsonParser parser = objectMapper.createParser(new NoCloseInputStream(zipIn));
@@ -313,7 +315,7 @@ private long importRowsFromParser(JsonParser parser,
313315

314316
log.info("[Migration/Import] Imported {} records for {}", total, entityClass.getSimpleName());
315317
if (listener != null) {
316-
listener.onProgress(new MigrationProgress(total, 0,
318+
listener.onProgress(MigrationProgress.of(total, 0,
317319
"Imported " + entityClass.getSimpleName() + " (" + total + " records)", total));
318320
}
319321

‎extensions/saas/sources/migration/src/main/java/tools/dynamia/modules/saas/migration/services/AccountMigrationJobServiceImpl.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import tools.dynamia.commons.logger.LoggingService;
1515
import tools.dynamia.domain.query.QueryConditions;
1616
import tools.dynamia.modules.saas.migration.api.MigrationProgressListener;
17+
import tools.dynamia.navigation.Page;
1718
import tools.jackson.databind.ObjectMapper;
1819
import org.springframework.beans.factory.annotation.Qualifier;
1920
import org.springframework.web.multipart.MultipartFile;
@@ -75,7 +76,7 @@ public class AccountMigrationJobServiceImpl implements AccountMigrationJobServic
7576

7677
private static final LoggingService log = LoggingService.get(AccountMigrationJobServiceImpl.class);
7778
private static final DateTimeFormatter FILE_TS = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss");
78-
private static final long PROGRESS_THROTTLE_MS = 500;
79+
private static final long PROGRESS_THROTTLE_MS = 2000;
7980

8081
/**
8182
* In-memory token registry: jobUuid → CancellationToken. Cleaned up when job finishes.
@@ -350,8 +351,7 @@ private MigrationProgressListener buildProgressListener(AccountMigrationJob job)
350351
AccountMigrationJob j = findByUuid(job.getUuid());
351352
if (j != null && !j.isFinished()) {
352353
if (isFirst) j.markRunning();
353-
j.updateProgress(p.percentage() >= 0 ? p.percentage() : j.getProgress(),
354-
p.message(), p.processedRecords());
354+
j.updateProgress(p);
355355
crudService.update(j);
356356
}
357357
});
@@ -364,6 +364,11 @@ private MigrationProgressListener buildProgressListener(AccountMigrationJob job)
364364
private Path buildOutputPath(AccountMigrationJob job) {
365365
String ts = LocalDateTime.now().format(FILE_TS);
366366
String fileName = "Account" + job.getAccountId() + "_" + ts + ".zip";
367+
try {
368+
Files.createDirectories(Paths.get(properties.getOutputDirectory()));
369+
} catch (IOException ignore) {
370+
371+
}
367372
return Paths.get(properties.getOutputDirectory(), fileName);
368373
}
369374

‎extensions/saas/sources/migration/src/test/java/tools/dynamia/modules/saas/migration/AccountMigrationJobTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
import org.junit.Assert;
1414
import org.junit.Test;
15+
import tools.dynamia.modules.saas.migration.api.MigrationProgress;
1516
import tools.dynamia.modules.saas.migration.domain.AccountJobStatus;
1617
import tools.dynamia.modules.saas.migration.domain.AccountMigrationJob;
1718

@@ -85,13 +86,13 @@ public void markCancelledStoresReason() {
8586
public void updateProgressClampsTo0_100Range() {
8687
AccountMigrationJob job = new AccountMigrationJob();
8788

88-
job.updateProgress(-5, "below zero", 0);
89+
job.updateProgress(MigrationProgress.of(-5L, 0L, "below zero", 0));
8990
Assert.assertEquals(0, job.getProgress());
9091

91-
job.updateProgress(150, "above hundred", 0);
92+
job.updateProgress(MigrationProgress.of(130, 5, "above hundred", 0));
9293
Assert.assertEquals(100, job.getProgress());
9394

94-
job.updateProgress(42, "normal", 0);
95+
job.updateProgress(MigrationProgress.of(42, 100, "normal", 0));
9596
Assert.assertEquals(42, job.getProgress());
9697
Assert.assertEquals("normal", job.getProgressMessage());
9798
}

0 commit comments

Comments
 (0)