Allow migration jobs to remove entities from DB without breaking their own pagination

Author: Toastbrot236

Refresh.Interfaces.Workers/Migrations/BackfillLevelAttributesMigration.cs

@@ -12,13 +12,14 @@ protected override IQueryable<GameLevel> SortAndFilter(IQueryable<GameLevel> que
             .OrderBy(l => l.LevelId);
     }
 
-    protected override void Migrate(WorkContext context, GameLevel[] batch)
+    protected override int Migrate(WorkContext context, GameLevel[] batch)
     {
         foreach (GameLevel level in batch)
         {
             context.Database.ApplyLevelMetadataFromAttributes(level);
         }
 
         context.Database.SaveChanges();
+        return batch.Length;
     }
 }

Refresh.Interfaces.Workers/Migrations/BackfillModdedPlanetFlagsMigration.cs

@@ -16,13 +16,14 @@ protected override IQueryable<GameUser> SortAndFilter(IQueryable<GameUser> query
             || u.BetaPlanetsHash != "0");
     }
 
-    protected override void Migrate(WorkContext context, GameUser[] batch)
+    protected override int Migrate(WorkContext context, GameUser[] batch)
     {
         foreach (GameUser user in batch)
         {
             context.Database.UpdatePlanetModdedStatus(user);
         }
 
         context.Database.SaveChanges();
+        return batch.Length;
     }
 }

Refresh.Interfaces.Workers/Migrations/BackfillReviewLabelsMigration.cs

@@ -6,7 +6,7 @@ namespace Refresh.Interfaces.Workers.Migrations;
 
 public class BackfillReviewLabelsMigration : MigrationJob<GameReview>
 {
-    protected override void Migrate(WorkContext context, GameReview[] batch)
+    protected override int Migrate(WorkContext context, GameReview[] batch)
     {
         foreach (GameReview review in batch)
         {
@@ -19,6 +19,7 @@ protected override void Migrate(WorkContext context, GameReview[] batch)
         }
 
         context.Database.SaveChanges();
+        return batch.Length;
     }
 
     protected override IQueryable<GameReview> SortAndFilter(IQueryable<GameReview> query)

Refresh.Interfaces.Workers/Migrations/BackfillRevisionMigration.cs

@@ -5,12 +5,13 @@ namespace Refresh.Interfaces.Workers.Migrations;
 
 public class BackfillRevisionMigration : MigrationJob<GameLevel>
 {
-    protected override void Migrate(WorkContext context, GameLevel[] batch)
+    protected override int Migrate(WorkContext context, GameLevel[] batch)
     {
         foreach (GameLevel level in batch)
         {
             context.Database.CreateRevisionForLevel(level, null);
         }
+        return batch.Length;
     }
 
     protected override IQueryable<GameLevel> SortAndFilter(IQueryable<GameLevel> query)

Refresh.Interfaces.Workers/Migrations/CalculateScoreRanksMigration.cs

@@ -12,12 +12,13 @@ protected override IQueryable<GameLevel> SortAndFilter(IQueryable<GameLevel> que
         return query.OrderBy(l => l.LevelId);
     }
 
-    protected override void Migrate(WorkContext context, GameLevel[] batch)
+    protected override int Migrate(WorkContext context, GameLevel[] batch)
     {
         foreach (GameLevel level in batch)
         {
             context.Database.RecalculateScoreStatistics(level);
         }
         context.Database.SaveChanges();
+        return batch.Length;
     }
 }

Refresh.Interfaces.Workers/Migrations/ClampPlayerLimitsMigration.cs

@@ -13,7 +13,7 @@ protected override IQueryable<GameLevel> SortAndFilter(IQueryable<GameLevel> que
         return query.OrderBy(l => l.LevelId);
     }
 
-    protected override void Migrate(WorkContext context, GameLevel[] batch)
+    protected override int Migrate(WorkContext context, GameLevel[] batch)
     {
         foreach (GameLevel level in batch)
         {
@@ -26,5 +26,7 @@ protected override void Migrate(WorkContext context, GameLevel[] batch)
             level.MinPlayers = Math.Clamp(level.MinPlayers, 1, 4);
             level.MaxPlayers = Math.Clamp(level.MaxPlayers, 1, 4);
         }
+
+        return batch.Length;
     }
 }

Refresh.Interfaces.Workers/Migrations/EnsureDeletedUsersDeletedMigration.cs

@@ -15,12 +15,13 @@ protected override IQueryable<GameUser> SortAndFilter(IQueryable<GameUser> query
             .OrderBy(u => u.UserId); // can't use join date here as we're changing the join date when we delete data
     }
 
-    protected override void Migrate(WorkContext context, GameUser[] batch)
+    protected override int Migrate(WorkContext context, GameUser[] batch)
     {
         foreach (GameUser user in batch)
         {
             context.Logger.LogWarning(RefreshContext.Worker, $"Deleting {user.Username}'s account again to ensure data has been wiped...");
             context.Database.DeleteUser(user);
         }
+        return batch.Length;
     }
 }

Refresh.Interfaces.Workers/Migrations/MoveSubjectsOutOfGamePhotosMigration.cs

@@ -12,7 +12,7 @@ protected override IQueryable<GamePhoto> SortAndFilter(IQueryable<GamePhoto> que
             .OrderBy(p => p.PhotoId);
     }
 
-    protected override void Migrate(WorkContext context, GamePhoto[] batch)
+    protected override int Migrate(WorkContext context, GamePhoto[] batch)
     {
         foreach (GamePhoto photo in batch)
         {
@@ -47,5 +47,6 @@ protected override void Migrate(WorkContext context, GamePhoto[] batch)
         }
 
         context.Database.SaveChanges();
+        return batch.Length;
     }
 }

Refresh.Workers/MigrationJob.cs

@@ -40,15 +40,21 @@ public override void ExecuteJob(WorkContext context)
 
         TEntity[] batch = query.ToArray();
 
-        Migrate(context, batch);
+        int entitiesLeftCount = Migrate(context, batch);
         context.Database.SaveChanges();
         transaction.Commit();
 
-        state.Processed += batch.Length;
+        state.Processed += entitiesLeftCount;
+        state.Total = state.Total - batch.Length + entitiesLeftCount;
         context.Logger.LogInfo(RefreshContext.Database, $"{this.JobId} migrated {batch.Length} objects ({state.Processed}/{state.Total}, complete: {state.Complete})");
     }
 
     protected abstract IQueryable<TEntity> SortAndFilter(IQueryable<TEntity> query);
 
-    protected abstract void Migrate(WorkContext context, TEntity[] batch);
+    /// <returns>
+    /// The number of entities in the batch, minus the number of entities removed from DB during this migration.
+    /// E.g. if 1000 items are in the given batch, and 5 got removed from DB during migration, 
+    /// the returned number will be 1000 - 5 = 995
+    /// </returns>
+    protected abstract int Migrate(WorkContext context, TEntity[] batch);
 }