extra flag parameter for storage-cli and logging stderr log stream (#4795)

* feat: debug flag added for storage-cli

* feat: request_id and user_guid added into logs as data

* fix: log_data removed

- we don't need to set since request_id already set in request.rb by either vcap_request_context_setter.rb or by logging_context_job.rb
- user_guid is also available in request.rb but only set by security_context_setter.rb so not available in jobs context

* fix: log level set to info

* feat: storage_cli_flag_options string added into config

* feat: storage_cli_flag_optionals added into schemas

* refactor: extra flag option paratmeter renamed to storage_cli_optional_flags

* refactor: cli_log_file function is removed

* test: unit tests for additional_flags function are added

* fix: extra debug logging and unused debug attribute are removed

Author: serdarozerr

config/cloud_controller.yml

@@ -325,6 +325,7 @@ storage_cli_config_file_droplets: config/storage_cli_config_droplets.json
 storage_cli_config_file_packages: config/storage_cli_config_packages.json
 storage_cli_config_file_buildpacks: config/storage_cli_config_buildpacks.json
 storage_cli_config_file_resource_pool: config/storage_cli_config_resource_pool.json
+storage_cli_optional_flags: ""
 
 newrelic_enabled: false
 

lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb

@@ -25,7 +25,7 @@ class StorageCliClient < BaseClient
         'Google' => 'gcs'
       }.freeze
 
-      IMPLEMENTED_PROVIDERS = %w[AzureRM aliyun].freeze
+      IMPLEMENTED_PROVIDERS = %w[AzureRM aliyun Google].freeze
 
       def initialize(directory_key:, resource_type:, root_dir:, min_size: nil, max_size: nil)
         raise 'Missing resource_type' if resource_type.nil?
@@ -167,11 +167,21 @@ def ensure_bucket_exists
 
       private
 
+      def additional_flags
+        flags_string = VCAP::CloudController::Config.config.get(:storage_cli_optional_flags)
+        return [] if flags_string.nil? || flags_string.empty?
+
+        flags_string.split
+      end
+
       def run_cli(command, *args, allow_exit_code_three: false)
-        logger.info("running storage-cli: #{@cli_path} -c #{@config_file} #{command} #{args.join(' ')}")
+        logger.info("running storage-cli: #{@cli_path} -s #{@storage_type} -c #{@config_file} #{additional_flags.join(' ')} #{command} #{args.join(' ')}")
 
         begin
-          stdout, stderr, status = Open3.capture3(@cli_path, '-s', @storage_type, '-c', @config_file, command, *args)
+          stdout, stderr, status = Open3.capture3(@cli_path, '-s', @storage_type, '-c', @config_file, *additional_flags, command, *args)
+          stderr.split("\n").each do |line|
+            logger.info("[INFO] storage-cli: #{line}")
+          end
         rescue StandardError => e
           raise BlobstoreError.new(e.inspect)
         end

lib/cloud_controller/config_schemas/api_schema.rb

@@ -100,6 +100,7 @@ class ApiSchema < VCAP::Config
           optional(:storage_cli_config_file_packages) => String,
           optional(:storage_cli_config_file_resource_pool) => String,
           optional(:storage_cli_config_file_droplets) => String,
+          optional(:storage_cli_optional_flags) => String,
 
           newrelic_enabled: bool,
 

lib/cloud_controller/config_schemas/clock_schema.rb

@@ -54,6 +54,7 @@ class ClockSchema < VCAP::Config
           optional(:storage_cli_config_file_packages) => String,
           optional(:storage_cli_config_file_resource_pool) => String,
           optional(:storage_cli_config_file_droplets) => String,
+          optional(:storage_cli_optional_flags) => String,
 
           newrelic_enabled: bool,
 

lib/cloud_controller/config_schemas/deployment_updater_schema.rb

@@ -19,6 +19,7 @@ class DeploymentUpdaterSchema < VCAP::Config
           optional(:storage_cli_config_file_packages) => String,
           optional(:storage_cli_config_file_resource_pool) => String,
           optional(:storage_cli_config_file_droplets) => String,
+          optional(:storage_cli_optional_flags) => String,
 
           readiness_port: {
             deployment_updater: Integer

lib/cloud_controller/config_schemas/worker_schema.rb

@@ -46,6 +46,7 @@ class WorkerSchema < VCAP::Config
           optional(:storage_cli_config_file_packages) => String,
           optional(:storage_cli_config_file_resource_pool) => String,
           optional(:storage_cli_config_file_droplets) => String,
+          optional(:storage_cli_optional_flags) => String,
 
           newrelic_enabled: bool,
 

spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb

@@ -49,7 +49,7 @@ module Blobstore
 
           expect do
             StorageCliClient.new(directory_key: 'dummy-key', root_dir: 'dummy-root', resource_type: 'droplets')
-          end.to raise_error(RuntimeError, 'Unimplemented provider: UnknownProvider, implemented ones are: AzureRM, aliyun')
+          end.to raise_error(RuntimeError, 'Unimplemented provider: UnknownProvider, implemented ones are: AzureRM, aliyun, Google')
 
           droplets_cfg.close!
         end
@@ -90,7 +90,7 @@ module Blobstore
             end
           end
 
-          allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil))
+          allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil, debug: nil))
         end
 
         after do
@@ -266,7 +266,7 @@ def build_client(resource_type)
               tmp_cfg.path
             end
           end
-          allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil))
+          allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil, debug: nil))
         end
 
         after { tmp_cfg.close! }
@@ -284,6 +284,28 @@ def build_client(resource_type)
         let(:deletable_blob) { StorageCliBlob.new('deletable-blob') }
         let(:dest_path) { File.join(Dir.mktmpdir, SecureRandom.uuid) }
 
+        describe 'optional flags' do
+          context 'when there is no extra flags' do
+            before do
+              allow(VCAP::CloudController::Config.config).to receive(:get).with(:storage_cli_optional_flags).and_return('')
+            end
+
+            it('returns empty list') {
+              expect(client.send(:additional_flags)).to eq([])
+            }
+          end
+
+          context 'when there is extra flags' do
+            before do
+              allow(VCAP::CloudController::Config.config).to receive(:get).with(:storage_cli_optional_flags).and_return('-log-level warn -log-file some/path/storage-cli.log')
+            end
+
+            it('returns empty list') {
+              expect(client.send(:additional_flags)).to eq(['-log-level', 'warn', '-log-file', 'some/path/storage-cli.log'])
+            }
+          end
+        end
+
         describe  '#exists?' do
           context 'when the blob exists' do
             before { allow(client).to receive(:run_cli).with('exists', any_args).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) }