Load delayed job metric plugin only for cc workers

The metrics plugin is only needed for the dedicated cc workers processes.

Additionally:
- Improve tests to avoid sporadic errors when metric is updated twice.
- Raise an error if `publish_metrics` is true and execution context is not set for cc workers.
  For the local workers there is no concept yet on how to publish metrics.

Author: johha

lib/cloud_controller/metrics/prometheus_updater.rb

@@ -103,7 +103,7 @@ def register_metrics_for_process
         METRICS.each { |metric| register(metric) }
         PUMA_METRICS.each { |metric| register(metric) }
       else
-        raise 'Could not register Prometheus metrics: Unknown execution context'
+        raise "Could not register Prometheus metrics: Unexpected execution context: #{@execution_context.inspect}"
       end
     end
     # rubocop:enable Metrics/CyclomaticComplexity

lib/delayed_job/delayed_worker.rb

@@ -27,6 +27,11 @@ def initialize(options)
   def start_working
     config = RakeConfig.config
     if @publish_metrics
+      # Local API workers don't have a concept of publishing metrics
+      unless VCAP::CloudController::ExecutionContext.from_process_type_env == VCAP::CloudController::ExecutionContext::CC_WORKER
+        raise 'Metric publishing is only supported for cc workers'
+      end
+
       setup_metrics(config)
       periodic_updater = CloudController::DependencyLocator.instance.vitals_periodic_updater
       periodic_updater.setup_updates
@@ -122,6 +127,8 @@ def setup_metrics(config)
     prometheus_dir = File.join(config.get(:directories, :tmpdir), 'prometheus')
     Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new(dir: prometheus_dir)
 
+    Delayed::Worker.plugins << DelayedJobMetrics::Plugin unless Delayed::Worker.plugins.include?(DelayedJobMetrics::Plugin)
+
     VCAP::CloudController::StandaloneMetricsWebserver.start_for_bosh_job(config.get(:prometheus_port) || 9394) if is_first_generic_worker_on_machine?
   end
 end

lib/delayed_job_plugins/delayed_jobs_metrics.rb

@@ -21,5 +21,3 @@ def prometheus
     end
   end
 end
-
-Delayed::Worker.plugins << DelayedJobMetrics::Plugin

spec/unit/lib/cloud_controller/metrics/prometheus_updater_spec.rb

@@ -34,7 +34,7 @@ module VCAP::CloudController::Metrics
 
       it 'raises error when execution context is nil' do
         allow(VCAP::CloudController::ExecutionContext).to receive(:from_process_type_env).and_return(nil)
-        expect { updater }.to raise_error('Could not register Prometheus metrics: Unknown execution context')
+        expect { updater }.to raise_error('Could not register Prometheus metrics: Unexpected execution context: nil')
       end
 
       context 'when execution context is set' do

spec/unit/lib/delayed_job/delayed_worker_spec.rb

@@ -214,16 +214,35 @@
         end
       end
 
+      context 'when set to true but not in CC_WORKER context' do
+        before do
+          options[:publish_metrics] = true
+          allow(VCAP::CloudController::ExecutionContext).to receive(:from_process_type_env).and_return(nil)
+        end
+
+        it 'raises an error' do
+          expect do
+            cc_delayed_worker.start_working
+          end.to raise_error('Metric publishing is only supported for cc workers')
+        end
+      end
+
       context 'when set to true' do
         before do
           options[:publish_metrics] = true
+          allow(VCAP::CloudController::ExecutionContext).to receive(:from_process_type_env).and_return(VCAP::CloudController::ExecutionContext::CC_WORKER)
         end
 
         it 'publishes metrics' do
           cc_delayed_worker.start_working
           expect(Prometheus::Client::DataStores::DirectFileStore).to have_received(:new)
         end
 
+        it 'loads the delayed job metrics plugin' do
+          cc_delayed_worker.start_working
+          expect(Delayed::Worker.plugins).to include(DelayedJobMetrics::Plugin)
+        end
+
         context 'when first worker on machine' do
           before do
             allow(cc_delayed_worker).to receive(:is_first_generic_worker_on_machine?).and_return(true)

spec/unit/lib/delayed_job_plugins/delayed_jobs_metrics_spec.rb

@@ -4,6 +4,7 @@
   let(:prometheus) { instance_double(VCAP::CloudController::Metrics::PrometheusUpdater) }
 
   before do
+    Delayed::Worker.plugins << DelayedJobMetrics::Plugin
     DelayedJobMetrics::Plugin.prometheus = prometheus
     allow(prometheus).to receive(:update_histogram_metric)
   end
@@ -33,13 +34,13 @@
         :cc_job_pickup_delay_seconds,
         be_within(0.5).of(10.0),
         labels: { queue: VCAP::CloudController::Jobs::Queues.generic, worker: 'test_worker' }
-      ).once
+      ).at_least(:once)
 
       expect(prometheus).to have_received(:update_histogram_metric).with(
         :cc_job_duration_seconds,
         kind_of(Numeric),
         labels: { queue: VCAP::CloudController::Jobs::Queues.generic, worker: 'test_worker' }
-      ).once
+      ).at_least(:once)
     end
   end
 end