migration

Author: valentijnscholten

dojo/db_migrations/0243_pghistory_models.py

@@ -3,6 +3,7 @@
 import django.contrib.auth.validators
 import django.core.validators
 import django.db.models.deletion
+import django.db.models.manager
 import django.utils.timezone
 import django_extensions.db.fields
 import dojo.models
@@ -919,4 +920,19 @@ class Migration(migrations.Migration):
             model_name='testevent',
             index=models.Index(fields=['pgh_context_id'], name='dojo_testev_pgh_con_e18502_idx'),
         ),
+        # DojoEvents proxy model for structured context field access
+        migrations.CreateModel(
+            name='DojoEvents',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+                'constraints': [],
+            },
+            bases=('pghistory.events',),
+            managers=[
+                ('no_objects', django.db.models.manager.Manager()),
+            ],
+        ),
     ]

dojo/management/commands/pghistory_backfill.py

@@ -149,6 +149,7 @@ def handle(self, *args, **options):
                     if not dry_run:
                         # Create events with preserved timestamps from original instances
                         event_records = []
+                        failed_records = []
                         for instance in filtered_batch:
                             try:
                                 # Create event record with all model fields
@@ -184,10 +185,12 @@ def handle(self, *args, **options):
                                 event_records.append(EventModel(**event_data))
 
                             except Exception as e:
+                                failed_records.append(instance.id)
                                 logger.error(
                                     f"Failed to prepare event for {model_name} "
                                     f"ID {instance.id}: {e}",
                                 )
+                                # Continue processing other records in the batch
 
                         # Bulk create all events in this batch
                         if event_records:
@@ -218,34 +221,49 @@ def handle(self, *args, **options):
                                 # Re-raise the exception instead of falling back
                                 raise
 
-                    processed += len(filtered_batch)
+                    # Count only successfully processed records
+                    successful_in_batch = len(event_records)
+                    failed_in_batch = len(failed_records)
+                    processed += successful_in_batch
+
+                    if failed_in_batch > 0:
+                        self.stdout.write(
+                            self.style.WARNING(
+                                f"  Batch {start + 1}-{end}: {successful_in_batch} successful, "
+                                f"{failed_in_batch} failed (IDs: {failed_records[:5]}{'...' if failed_in_batch > 5 else ''})",
+                            ),
+                        )
+
                     self.stdout.write(
                         f"  Processed {processed:,}/{backfill_count:,} records needing backfill "
                         f"({processed / backfill_count * 100:.1f}%)",
                     )
 
                 total_processed += processed
-                self.stdout.write(
-                    self.style.SUCCESS(
-                        f"  ✓ Completed {model_name}: {processed:,} records",
-                    ),
-                )
+
+                # Show completion summary
+                if processed < backfill_count:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"  ⚠ Completed {model_name}: {processed:,}/{backfill_count:,} records "
+                            f"({backfill_count - processed:,} records failed and will need to be retried)",
+                        ),
+                    )
+                else:
+                    self.stdout.write(
+                        self.style.SUCCESS(
+                            f"  ✓ Completed {model_name}: {processed:,} records",
+                        ),
+                    )
 
             except Exception as e:
                 self.stdout.write(
                     self.style.ERROR(f"  ✗ Failed to process {model_name}: {e}"),
                 )
                 logger.error(f"Error processing {model_name}: {e}")
 
-        if dry_run:
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"\nDRY RUN COMPLETE: Would have processed {total_processed:,} records",
-                ),
-            )
-        else:
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"\nBACKFILL COMPLETE: Processed {total_processed:,} records",
-                ),
-            )
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\nBACKFILL COMPLETE: Processed {total_processed:,} records",
+            ),
+        )

dojo/pghistory_models.py

@@ -5,7 +5,7 @@
 to expose context fields as structured fields.
 
 Note: Performance indexes for the pghistory_context table are managed
-via Django migration 0245_add_pghistory_context_indexes.py rather than
+via Django migration 0244_pghistory_indices.py rather than
 through model Meta classes, since the context table is managed by
 the pghistory library itself.
 """
@@ -28,3 +28,4 @@ class DojoEvents(pghistory.models.Events):
 
     class Meta:
         proxy = True
+        app_label = "dojo"