Prevent broker stuck in state SYNCHRONIZING when db is down before the job starts

Author: kathap

app/actions/v3/service_broker_update.rb

@@ -45,8 +45,6 @@ def enqueue_update
         pollable_job = nil
         previous_broker_state = broker.state
         ServiceBrokerUpdateRequest.db.transaction do
-          broker.update(state: ServiceBrokerStateEnum::SYNCHRONIZING)
-
           update_request = ServiceBrokerUpdateRequest.create(params)
           MetadataUpdate.update(update_request, message, destroy_nil: false)
 

app/jobs/v3/services/update_broker_job.rb

@@ -47,6 +47,7 @@ class Perform
         def initialize(update_request_guid, previous_broker_state, user_audit_info:)
           @update_request_guid = update_request_guid
           @update_request = ServiceBrokerUpdateRequest.find(guid: update_request_guid)
+
           raise 'Service broker update request not found' if @update_request.nil?
 
           @broker = ServiceBroker.find(id: @update_request.service_broker_id)
@@ -57,6 +58,8 @@ def initialize(update_request_guid, previous_broker_state, user_audit_info:)
         def perform
           raise CloudController::Errors::V3::ApiError.new_from_details('ServiceBrokerGone') if broker.nil?
 
+          broker.update(state: ServiceBrokerStateEnum::SYNCHRONIZING)
+
           ServiceBroker.db.transaction do
             broker.update(update_params)
 

spec/unit/actions/v3/service_broker_update_spec.rb

@@ -166,10 +166,24 @@ module CloudController
           ).once
         end
 
-        it 'sets the state to SYNCHRONIZING' do
+        it 'does not immediately set the state to SYNCHRONIZING' do
           action.enqueue_update
 
-          expect(existing_service_broker.state).to eq(ServiceBrokerStateEnum::SYNCHRONIZING)
+          expect(existing_service_broker.state).to eq(ServiceBrokerStateEnum::AVAILABLE)
+        end
+
+        it 'sets the state to SYNCHRONIZING when the job runs' do
+          action.enqueue_update
+
+          broker_state_during_execution = nil
+          allow_any_instance_of(VCAP::CloudController::V3::ServiceBrokerCatalogUpdater).to receive(:refresh) do
+            broker_state_during_execution = existing_service_broker.reload.state
+            nil
+          end
+
+          execute_all_jobs(expected_successes: 1, expected_failures: 0)
+          expect(broker_state_during_execution).to eq(ServiceBrokerStateEnum::SYNCHRONIZING)
+          expect(existing_service_broker.reload.state).to eq(ServiceBrokerStateEnum::AVAILABLE)
         end
 
         it 'creates an audit event' do
@@ -237,10 +251,24 @@ module CloudController
         context 'legacy service brokers' do
           let!(:state) { '' } # broker creating from V2 endpoint will not have a state
 
-          it 'sets the state to SYNCHRONIZING' do
+          it 'does not immediately set the state to SYNCHRONIZING' do
             action.enqueue_update
 
-            expect(existing_service_broker.reload.state).to eq(ServiceBrokerStateEnum::SYNCHRONIZING)
+            expect(existing_service_broker.reload.state).to eq('')
+          end
+
+          it 'sets the state to SYNCHRONIZING when the job runs' do
+            action.enqueue_update
+
+            broker_state_during_execution = nil
+            allow_any_instance_of(VCAP::CloudController::V3::ServiceBrokerCatalogUpdater).to receive(:refresh) do
+              broker_state_during_execution = existing_service_broker.reload.state
+              nil
+            end
+
+            execute_all_jobs(expected_successes: 1, expected_failures: 0)
+            expect(broker_state_during_execution).to eq(ServiceBrokerStateEnum::SYNCHRONIZING)
+            expect(existing_service_broker.reload.state).to eq(ServiceBrokerStateEnum::AVAILABLE)
           end
 
           it 'creates and returns a synchronization job' do