fix(nimbus): prevent kinto preview sync from clobbering concurrent user state transitions (#15619)

Because

* The kinto preview sync task hydrated experiments in memory across slow
Kinto network calls, then called experiment.save(), clobbering any
concurrent write — most visibly a Request Launch on a Preview
experiment, which appeared to succeed in the HTMX response but reverted
on refresh
* All three sections (push, expire, unpublish) had the same lost-update
shape

This commit

* Replaces experiment.save() with conditional QuerySet.update() in all
three sections, gating each write on the status the task observed and
writing only the columns it owns
* Adds regression tests for the push and expiry races

Fixes #15618

Author: jaredlockhart

experimenter/experimenter/kinto/tasks.py

@@ -505,10 +505,22 @@ def nimbus_synchronize_preview_experiments_in_kinto():
             kinto_client.create_record(data)
 
             with transaction.atomic():
-                experiment.published_dto = data
-                experiment.published_date = timezone.now()
-                experiment.save()
+                updated = NimbusExperiment.objects.filter(
+                    pk=experiment.pk,
+                    status=NimbusExperiment.Status.PREVIEW,
+                ).update(
+                    published_dto=data,
+                    published_date=timezone.now(),
+                )
+
+                if not updated:
+                    logger.info(
+                        f"{experiment.slug} left Preview during push; "
+                        f"next sync will reconcile Kinto"
+                    )
+                    continue
 
+                experiment.refresh_from_db()
                 generate_nimbus_changelog(
                     experiment,
                     get_kinto_user(),
@@ -525,11 +537,20 @@ def nimbus_synchronize_preview_experiments_in_kinto():
 
         for experiment in expired_experiments:
             with transaction.atomic():
-                experiment.status = NimbusExperiment.Status.DRAFT
-                experiment.publish_status = NimbusExperiment.PublishStatus.IDLE
-                experiment.published_date = None
-                experiment.save()
+                updated = NimbusExperiment.objects.filter(
+                    pk=experiment.pk,
+                    status=NimbusExperiment.Status.PREVIEW,
+                ).update(
+                    status=NimbusExperiment.Status.DRAFT,
+                    publish_status=NimbusExperiment.PublishStatus.IDLE,
+                    published_date=None,
+                )
+
+                if not updated:
+                    logger.info(f"{experiment.slug} left Preview before expiry; skipping")
+                    continue
 
+                experiment.refresh_from_db()
                 generate_nimbus_changelog(
                     experiment,
                     get_kinto_user(),
@@ -545,12 +566,15 @@ def nimbus_synchronize_preview_experiments_in_kinto():
             kinto_client.delete_record(experiment.slug)
 
             with transaction.atomic():
-                experiment.refresh_from_db()
-                if experiment.status == NimbusExperiment.Status.DRAFT:
-                    experiment.published_date = None
-                    experiment.published_dto = None
-                    experiment.save()
+                NimbusExperiment.objects.filter(
+                    pk=experiment.pk,
+                    status=NimbusExperiment.Status.DRAFT,
+                ).update(
+                    published_date=None,
+                    published_dto=None,
+                )
 
+                experiment.refresh_from_db()
                 generate_nimbus_changelog(
                     experiment,
                     get_kinto_user(),

experimenter/experimenter/kinto/tests/test_tasks.py

@@ -1666,6 +1666,65 @@ def test_reraises_exception(self):
         with self.assertRaises(Exception):
             tasks.nimbus_synchronize_preview_experiments_in_kinto()
 
+    def test_does_not_revert_user_transition_during_concurrent_preview_push(self):
+        experiment = NimbusExperimentFactory.create_with_lifecycle(
+            NimbusExperimentFactory.Lifecycles.PREVIEW,
+            published_date=None,
+            application=NimbusExperiment.Application.DESKTOP,
+        )
+        self.setup_kinto_get_main_records([])
+
+        def simulate_user_request_launch(*args, **kwargs):
+            NimbusExperiment.objects.filter(pk=experiment.pk).update(
+                status=NimbusExperiment.Status.DRAFT,
+                status_next=NimbusExperiment.Status.LIVE,
+                publish_status=NimbusExperiment.PublishStatus.REVIEW,
+            )
+
+        self.mock_kinto_client.create_record.side_effect = simulate_user_request_launch
+
+        tasks.nimbus_synchronize_preview_experiments_in_kinto()
+
+        experiment.refresh_from_db()
+        self.assertEqual(experiment.status, NimbusExperiment.Status.DRAFT)
+        self.assertEqual(experiment.status_next, NimbusExperiment.Status.LIVE)
+        self.assertEqual(experiment.publish_status, NimbusExperiment.PublishStatus.REVIEW)
+        self.assertFalse(
+            experiment.changes.filter(
+                message=NimbusChangeLog.Messages.PUSHED_TO_PREVIEW,
+                changed_by__email=settings.KINTO_DEFAULT_CHANGELOG_USER,
+            ).exists()
+        )
+
+    def test_does_not_expire_experiment_no_longer_in_preview(self):
+        experiment = NimbusExperimentFactory.create_with_lifecycle(
+            NimbusExperimentFactory.Lifecycles.PREVIEW,
+            published_date=timezone.now(),
+            application=NimbusExperiment.Application.DESKTOP,
+        )
+
+        NimbusExperiment.objects.filter(id=experiment.id).update(
+            _updated_date_time=timezone.now() - timezone.timedelta(days=31),
+            status=NimbusExperiment.Status.DRAFT,
+            status_next=NimbusExperiment.Status.LIVE,
+            publish_status=NimbusExperiment.PublishStatus.REVIEW,
+        )
+
+        self.setup_kinto_get_main_records([experiment.slug])
+
+        tasks.nimbus_synchronize_preview_experiments_in_kinto()
+
+        experiment.refresh_from_db()
+        self.assertEqual(experiment.status, NimbusExperiment.Status.DRAFT)
+        self.assertEqual(experiment.status_next, NimbusExperiment.Status.LIVE)
+        self.assertEqual(experiment.publish_status, NimbusExperiment.PublishStatus.REVIEW)
+        self.assertFalse(
+            experiment.changes.filter(
+                message=NimbusChangeLog.Messages.EXPIRED_FROM_PREVIEW,
+                changed_by__email=settings.KINTO_DEFAULT_CHANGELOG_USER,
+            ).exists()
+        )
+
 
 class TestNimbusSendEmails(MockKintoClientMixin, TestCase):
     def setUp(self):