|
| 1 | +module VCAP::CloudController |
| 2 | + module Jobs |
| 3 | + module Runtime |
| 4 | + class ServiceOperationsCreateInProgressCleanup < VCAP::CloudController::Jobs::CCJob |
| 5 | + BATCH_SIZE = 10 |
| 6 | + |
| 7 | + def perform |
| 8 | + logger.info("Cleaning up service 'create' operations stuck in 'in progress'") |
| 9 | + cleanup_operations(ServiceInstanceOperation, ServiceInstance, :service_instance_id, 'service_instance.create', :cleanup_failed_provision) |
| 10 | + cleanup_operations(ServiceBindingOperation, ServiceBinding, :service_binding_id, 'service_bindings.create', :cleanup_failed_bind) |
| 11 | + cleanup_operations(ServiceKeyOperation, ServiceKey, :service_key_id, 'service_keys.create', :cleanup_failed_key) |
| 12 | + end |
| 13 | + |
| 14 | + def max_attempts |
| 15 | + 1 |
| 16 | + end |
| 17 | + |
| 18 | + private |
| 19 | + |
| 20 | + def cleanup_operations(operation_model, instance_model, foreign_key, jobs_operation, orphan_mitigator_method) |
| 21 | + # The explanation below uses service_instance_operations as the concrete example; |
| 22 | + # the same logic applies to service_binding_operations and service_key_operations |
| 23 | + # when invoked with their respective arguments. |
| 24 | + # |
| 25 | + # Find stuck service instance 'in progress' operations where the broker is still working |
| 26 | + # but CC's polling job has permanently failed due to a transient error (e.g. brief db connection flip). |
| 27 | + # Join path: service_instance_operations → service_instances → jobs → delayed_jobs. |
| 28 | + # |
| 29 | + # Filters: |
| 30 | + # - service_instance_operations.state='in progress': the broker has not yet reported a final state |
| 31 | + # (succeeded or failed) that CC could successfully persist; if CC had received and saved a final |
| 32 | + # state from the broker, this column would already be 'succeeded' or 'failed' — not 'in progress' |
| 33 | + # - service_instance_operations.type='create': scope to create operations only |
| 34 | + # - service_instance_operations.created_at > CURRENT_TIMESTAMP - max_duration: operations beyond the max async polling window |
| 35 | + # are intentionally excluded — the broker has given up on them too, so they are out of scope for this cleanup |
| 36 | + # - jobs.state IN (POLLING, FAILED): the pollable job has not reached COMPLETE (a successful job |
| 37 | + # would already be done and is out of scope); POLLING covers the case where the failure hook |
| 38 | + # itself couldn't write FAILED due to the DB flip |
| 39 | + # - jobs.operation='service_instance.create': prevents matching update/delete jobs for the same |
| 40 | + # service instance that happen to share the same resource_guid |
| 41 | + # - delayed_jobs.failed_at IS NOT NULL: the delayed job permanently failed (exhausted max_attempts); |
| 42 | + # jobs still alive or locked have failed_at=NULL and must not be touched |
| 43 | + operation_table = operation_model.table_name |
| 44 | + instance_table = instance_model.table_name |
| 45 | + |
| 46 | + stuck = operation_model. |
| 47 | + join(instance_table, id: Sequel[operation_table][foreign_key]). |
| 48 | + join(:jobs, resource_guid: Sequel[instance_table][:guid]). |
| 49 | + join(:delayed_jobs, guid: Sequel[:jobs][:delayed_job_guid]). |
| 50 | + where(Sequel[operation_table][:state] => 'in progress'). |
| 51 | + where(Sequel[operation_table][:type] => 'create'). |
| 52 | + where(Sequel.lit("#{operation_table}.created_at > CURRENT_TIMESTAMP - INTERVAL '?' SECOND", default_maximum_duration_seconds.to_i)). |
| 53 | + where(Sequel[:jobs][:state] => [PollableJobModel::POLLING_STATE, PollableJobModel::FAILED_STATE]). |
| 54 | + where(Sequel[:jobs][:operation] => jobs_operation). |
| 55 | + exclude(Sequel[:delayed_jobs][:failed_at] => nil). |
| 56 | + select( |
| 57 | + Sequel[:jobs][:guid].as(:pollable_guid), |
| 58 | + Sequel[operation_table][:id].as(:op_id), |
| 59 | + Sequel[operation_table][foreign_key].as(:resource_id) |
| 60 | + ). |
| 61 | + order(Sequel[operation_table][:created_at]). |
| 62 | + limit(BATCH_SIZE) |
| 63 | + |
| 64 | + stuck.each do |row| |
| 65 | + mitigate_orphan(operation_model, instance_model, orphan_mitigator_method, |
| 66 | + row[:op_id], row[:resource_id], row[:pollable_guid]) |
| 67 | + end |
| 68 | + end |
| 69 | + |
| 70 | + def mitigate_orphan(operation_model, instance_model, orphan_mitigator_method, op_id, resource_id, pollable_guid) |
| 71 | + # Mark the stuck create operation as failed, mark its pollable job as failed, |
| 72 | + # and trigger broker-side orphan deprovisioning to clean up any resource the |
| 73 | + # broker may have created. |
| 74 | + operation_model.db.transaction do |
| 75 | + operation = operation_model.where(id: op_id, state: 'in progress').for_update.skip_locked.first |
| 76 | + return unless operation |
| 77 | + |
| 78 | + instance = instance_model.first(id: resource_id) |
| 79 | + return unless instance |
| 80 | + |
| 81 | + instance_type = instance_model.to_s.split('::').last |
| 82 | + |
| 83 | + logger.info( |
| 84 | + "#{instance_type} #{instance.guid} create operation is stuck in 'in progress'. " \ |
| 85 | + "Setting operation's state to 'failed' and pollable job's state to 'FAILED'.", |
| 86 | + instance_type: instance_type, |
| 87 | + instance_guid: instance.guid, |
| 88 | + operation_id: op_id, |
| 89 | + pollable_job_guid: pollable_guid |
| 90 | + ) |
| 91 | + |
| 92 | + operation.update(state: 'failed', |
| 93 | + description: "Operation was stuck in 'in progress' state. Set to 'failed' by cleanup job; orphan mitigation triggered.") |
| 94 | + PollableJobModel.where(guid: pollable_guid).update(state: PollableJobModel::FAILED_STATE) |
| 95 | + orphan_mitigator.send(orphan_mitigator_method, instance) |
| 96 | + end |
| 97 | + end |
| 98 | + |
| 99 | + def orphan_mitigator |
| 100 | + @orphan_mitigator ||= VCAP::Services::ServiceBrokers::V2::OrphanMitigator.new |
| 101 | + end |
| 102 | + |
| 103 | + def default_maximum_duration_seconds |
| 104 | + Config.config.get(:broker_client_max_async_poll_duration_minutes).minutes |
| 105 | + end |
| 106 | + |
| 107 | + def logger |
| 108 | + @logger ||= Steno.logger('cc.background.service-operations-create-in-progress-cleanup') |
| 109 | + end |
| 110 | + |
| 111 | + def job_name_in_configuration |
| 112 | + :service_operations_create_in_progress_cleanup |
| 113 | + end |
| 114 | + end |
| 115 | + end |
| 116 | + end |
| 117 | +end |
0 commit comments