Exclude models when FKs have already been excluded

Author: bjester

morango/sync/operations.py

@@ -512,6 +512,12 @@ def _deserialize_from_store(profile, skip_erroring=False, filter=None):
                         if app_model:
                             app_models.append(app_model)
                         for fk_model, fk_refs in model_deferred_fks.items():
+                            # validate that the FK references aren't to anything already in the
+                            # excluded list, which should only contain models which failed to
+                            # deserialize for reasons other than broken FKs at this point
+                            for fk_ref in fk_refs:
+                                if fk_ref.to_pk in excluded_list:
+                                    raise exceptions.ValidationError("{} with id {} failed to deserialize".format(fk_model, fk_ref.to_pk))
                             deferred_fks[fk_model].extend(fk_refs)
                     except (
                         exceptions.ValidationError,
@@ -533,17 +539,23 @@ def _deserialize_from_store(profile, skip_erroring=False, filter=None):
                 # array for holding db values from the fields of each model for this class
                 db_values = []
                 for app_model in app_models:
-                    print("CHECK EXCLUDE", app_model.pk, excluded_list)
                     if (
                         app_model.pk not in excluded_list
                         and app_model.pk not in deleted_list
                     ):
-                        new_db_values = []
-                        for f in fields:
-                            value = getattr(app_model, f.attname)
-                            db_value = f.get_db_prep_value(value, connection)
-                            new_db_values.append(db_value)
-                        db_values += new_db_values
+                        # handle any errors that might come from `get_db_prep_value`
+                        try:
+                            new_db_values = []
+                            for f in fields:
+                                value = getattr(app_model, f.attname)
+                                db_value = f.get_db_prep_value(value, connection)
+                                new_db_values.append(db_value)
+                            db_values += new_db_values
+                        except ValueError as e:
+                            excluded_list.append(app_model.pk)
+                            store_model = store_models.get(pk=app_model.pk)
+                            store_model.deserialization_error = str(e)
+                            store_model.save(update_fields=["deserialization_error"])
 
                 if db_values:
                     with connection.cursor() as cursor:

tests/testapp/tests/sync/test_operations.py

@@ -2,15 +2,14 @@
 import uuid
 
 import factory
+import mock
 from django.db import connection
-from django.test import TestCase
 from django.test import override_settings
+from django.test import TestCase
 from django.utils import timezone
 from facility_profile.models import Facility
 from facility_profile.models import MyUser
 from facility_profile.models import SummaryLog
-import mock
-import pytest
 
 from ..helpers import create_buffer_and_store_dummy_data
 from ..helpers import create_dummy_store_data
@@ -19,12 +18,12 @@
 from morango.errors import MorangoLimitExceeded
 from morango.models.core import Buffer
 from morango.models.core import DatabaseIDModel
+from morango.models.core import DatabaseMaxCounter
 from morango.models.core import InstanceIDModel
 from morango.models.core import RecordMaxCounter
 from morango.models.core import RecordMaxCounterBuffer
 from morango.models.core import Store
 from morango.models.core import SyncSession
-from morango.models.core import DatabaseMaxCounter
 from morango.models.core import TransferSession
 from morango.sync.backends.utils import load_backend
 from morango.sync.context import LocalSessionContext
@@ -35,11 +34,11 @@
 from morango.sync.operations import _queue_into_buffer_v1
 from morango.sync.operations import _queue_into_buffer_v2
 from morango.sync.operations import CleanupOperation
-from morango.sync.operations import ReceiverDequeueOperation
-from morango.sync.operations import ProducerDequeueOperation
-from morango.sync.operations import ReceiverDeserializeOperation
 from morango.sync.operations import InitializeOperation
+from morango.sync.operations import ProducerDequeueOperation
 from morango.sync.operations import ProducerQueueOperation
+from morango.sync.operations import ReceiverDequeueOperation
+from morango.sync.operations import ReceiverDeserializeOperation
 from morango.sync.operations import ReceiverQueueOperation
 from morango.sync.syncsession import TransferClient
 
@@ -1101,25 +1100,25 @@ def test_successful_deserialization(self):
         self.assert_deserialization()
 
     def test_deserialization_with_missing_username(self):
-            
+
         self.serialized_user["username"] = ""
 
         self.serialize_all_to_store()
 
         _deserialize_from_store(self.profile)
 
         self.assert_deserialization(user_deserialized=False, log1_deserialized=False, log2_deserialized=False)
-    
+
     def test_deserialization_with_excessively_long_username(self):
-            
+
         self.serialized_user["username"] = "a" * 256
 
         self.serialize_all_to_store()
 
         _deserialize_from_store(self.profile)
 
         self.assert_deserialization(user_deserialized=False, log1_deserialized=False, log2_deserialized=False)
-        
+
     def test_deserialization_with_invalid_content_id(self):
 
         self.serialized_log1["content_id"] = "invalid"