prepare ccng for blobstore benchmark errand

Author: kathap

lib/cloud_controller/benchmark/blobstore.rb

@@ -14,12 +14,35 @@ def perform
         zip_output_dir = Dir.mktmpdir
         zip_file = zip_resources(resource_dir, zip_output_dir)
 
-        benchmark_packages(zip_file, resource_dir)
-        benchmark_buildpacks(resource_dir)
-        benchmark_droplets(zip_file, resource_dir)
-        benchmark_big_droplets(resource_dir)
+        package_guid, resource_timing = upload_package(zip_file)
+        puts("package upload timing: #{resource_timing * 1000}ms")
+
+        resource_timing = download_package(package_guid, resource_dir)
+        puts("package download timing: #{resource_timing * 1000}ms")
+
+        bytes_read, resource_timing = download_buildpacks(resource_dir)
+        puts("downloaded #{Buildpack.count} buildpacks, total #{bytes_read} bytes read")
+        puts("buildpack download timing: #{resource_timing * 1000}ms")
+
+        droplet_guid, resource_timing = upload_droplet(zip_file)
+        puts("droplet upload timing: #{resource_timing * 1000}ms")
+
+        resource_timing = download_droplet(droplet_guid, resource_dir)
+        puts("droplet download timing: #{resource_timing * 1000}ms")
+
+        big_droplet_file = Tempfile.new('big-droplet', resource_dir)
+        big_droplet_file.write('abc' * 1024 * 1024 * 100)
+        big_droplet_guid, resource_timing = upload_droplet(big_droplet_file.path)
+        puts("big droplet upload timing: #{resource_timing * 1000}ms")
+
+        resource_timing = download_droplet(big_droplet_guid, resource_dir)
+        puts("big droplet download timing: #{resource_timing * 1000}ms")
       ensure
-        cleanup(resource_dir, zip_output_dir)
+        FileUtils.remove_dir(resource_dir, true)
+        FileUtils.remove_dir(zip_output_dir, true)
+        package_blobstore_client.delete(package_guid) if package_guid
+        droplet_blobstore_client.delete(droplet_guid) if droplet_guid
+        droplet_blobstore_client.delete(big_droplet_guid) if big_droplet_guid
       end
 
       def resource_match(dir_path)
@@ -32,125 +55,43 @@ def resource_match(dir_path)
         end
       end
 
-      def upload_package(package_path, client=package_blobstore_client)
-        copy_to_blobstore(package_path, client)
+      def upload_package(package_path)
+        copy_to_blobstore(package_path, package_blobstore_client)
       end
 
-      def download_package(package_guid, tmp_dir, client=package_blobstore_client)
+      def download_package(package_guid, tmp_dir)
         tempfile = Tempfile.new('package-download-benchmark', tmp_dir)
         ::Benchmark.realtime do
-          client.download_from_blobstore(package_guid, tempfile.path)
+          package_blobstore_client.download_from_blobstore(package_guid, tempfile.path)
         end
       end
 
-      def download_buildpacks(tmp_dir, client=buildpack_blobstore_client)
+      def download_buildpacks(tmp_dir)
         tempfile = Tempfile.new('buildpack-download-benchmark', tmp_dir)
         bytes_read = 0
 
         timing = ::Benchmark.realtime do
           bytes_read = Buildpack.map do |buildpack|
-            client.download_from_blobstore(buildpack.key, tempfile.path)
+            buildpack_blobstore_client.download_from_blobstore(buildpack.key, tempfile.path)
             File.stat(tempfile.path).size
           end.sum
         end
 
         [bytes_read, timing]
       end
 
-      def upload_droplet(droplet_path, client=droplet_blobstore_client)
-        copy_to_blobstore(droplet_path, client)
+      def upload_droplet(droplet_path)
+        copy_to_blobstore(droplet_path, droplet_blobstore_client)
       end
 
-      def download_droplet(droplet_guid, tmp_dir, client=droplet_blobstore_client)
+      def download_droplet(droplet_guid, tmp_dir)
         tempfile = Tempfile.new('droplet-download-benchmark', tmp_dir)
 
         ::Benchmark.realtime do
-          client.download_from_blobstore(droplet_guid, tempfile.path)
-        end
-      end
-
-      def benchmark_packages(zip_file, resource_dir)
-        fog_guid, fog_time = upload_package(zip_file, package_blobstore_client)
-        cli_guid, cli_time = upload_package(zip_file, package_blobstore_client_storage_cli)
-
-        puts("package upload timing fog: #{fog_time * 1000}ms")
-        puts("package upload timing storage-cli: #{cli_time * 1000}ms")
-
-        fog_dl = download_package(fog_guid, resource_dir, package_blobstore_client)
-        cli_dl = download_package(cli_guid, resource_dir, package_blobstore_client_storage_cli)
-
-        puts("package download timing fog: #{fog_dl * 1000}ms")
-        puts("package download timing storage-cli: #{cli_dl * 1000}ms")
-
-        remember_cleanup(:package, fog_guid, cli_guid)
-      end
-
-      def benchmark_buildpacks(resource_dir)
-        fog_bytes, fog_time = download_buildpacks(resource_dir, buildpack_blobstore_client)
-        cli_bytes, cli_time = download_buildpacks(resource_dir, buildpack_blobstore_client_storage_cli)
-
-        puts("downloaded #{Buildpack.count} buildpacks, total fog #{fog_bytes} bytes read")
-        puts("downloaded #{Buildpack.count} buildpacks, total storage-cli #{cli_bytes} bytes read")
-        puts("buildpack download timing fog: #{fog_time * 1000}ms")
-        puts("buildpack download timing storage-cli: #{cli_time * 1000}ms")
-      end
-
-      def benchmark_droplets(zip_file, resource_dir)
-        fog_guid, fog_time = upload_droplet(zip_file, droplet_blobstore_client)
-        cli_guid, cli_time = upload_droplet(zip_file, droplet_blobstore_client_storage_cli)
-
-        puts("droplet upload timing fog: #{fog_time * 1000}ms")
-        puts("droplet upload timing storage-cli: #{cli_time * 1000}ms")
-
-        fog_dl = download_droplet(fog_guid, resource_dir, droplet_blobstore_client)
-        cli_dl = download_droplet(cli_guid, resource_dir, droplet_blobstore_client_storage_cli)
-
-        puts("droplet download timing fog: #{fog_dl * 1000}ms")
-        puts("droplet download timing storage-cli: #{cli_dl * 1000}ms")
-
-        remember_cleanup(:droplet, fog_guid, cli_guid)
-      end
-
-      def benchmark_big_droplets(resource_dir)
-        fog_guid, fog_time = upload_big_droplet(resource_dir, droplet_blobstore_client)
-        cli_guid, cli_time = upload_big_droplet(resource_dir, droplet_blobstore_client_storage_cli)
-
-        puts("big droplet upload timing fog: #{fog_time * 1000}ms")
-        puts("big droplet upload timing storage-cli: #{cli_time * 1000}ms")
-
-        fog_dl = download_droplet(fog_guid, resource_dir, droplet_blobstore_client)
-        cli_dl = download_droplet(cli_guid, resource_dir, droplet_blobstore_client_storage_cli)
-
-        puts("big droplet download timing fog: #{fog_dl * 1000}ms")
-        puts("big droplet download timing storage-cli: #{cli_dl * 1000}ms")
-
-        remember_cleanup(:droplet, fog_guid, cli_guid)
-      end
-
-      def remember_cleanup(type, fog_guid, cli_guid)
-        cleanup_items << [type, fog_guid, cli_guid]
-      end
-
-      def cleanup(resource_dir, zip_output_dir)
-        FileUtils.remove_dir(resource_dir, true)
-        FileUtils.remove_dir(zip_output_dir, true)
-
-        cleanup_items.each do |type, fog_guid, cli_guid|
-          client_fog, client_cli =
-            case type
-            when :package then [package_blobstore_client, package_blobstore_client_storage_cli]
-            when :droplet then [droplet_blobstore_client, droplet_blobstore_client_storage_cli]
-            end
-
-          client_fog.delete(fog_guid) if fog_guid
-          client_cli.delete(cli_guid) if cli_guid
+          droplet_blobstore_client.download_from_blobstore(droplet_guid, tempfile.path)
         end
       end
 
-      def cleanup_items
-        @cleanup_items ||= []
-      end
-
       private
 
       def generate_resources
@@ -187,27 +128,15 @@ def copy_to_blobstore(path, client)
       end
 
       def buildpack_blobstore_client
-        @buildpack_blobstore_client ||= CloudController::DependencyLocator.instance.buildpack_blobstore(blobstore_type: 'fog')
-      end
-
-      def buildpack_blobstore_client_storage_cli
-        @buildpack_blobstore_client_storage_cli ||= CloudController::DependencyLocator.instance.buildpack_blobstore(blobstore_type: 'storage-cli')
+        @buildpack_blobstore_client ||= CloudController::DependencyLocator.instance.buildpack_blobstore
       end
 
       def droplet_blobstore_client
-        @droplet_blobstore_client ||= CloudController::DependencyLocator.instance.droplet_blobstore(blobstore_type: 'fog')
-      end
-
-      def droplet_blobstore_client_storage_cli
-        @droplet_blobstore_client_storage_cli ||= CloudController::DependencyLocator.instance.droplet_blobstore(blobstore_type: 'storage-cli')
+        @droplet_blobstore_client ||= CloudController::DependencyLocator.instance.droplet_blobstore
       end
 
       def package_blobstore_client
-        @package_blobstore_client ||= CloudController::DependencyLocator.instance.package_blobstore(blobstore_type: 'fog')
-      end
-
-      def package_blobstore_client_storage_cli
-        @package_blobstore_client_storage_cli ||= CloudController::DependencyLocator.instance.package_blobstore(blobstore_type: 'storage-cli')
+        @package_blobstore_client ||= CloudController::DependencyLocator.instance.package_blobstore
       end
 
       def resource_pool

lib/cloud_controller/config.rb

@@ -12,6 +12,7 @@
 require 'cloud_controller/config_schemas/worker_schema'
 require 'cloud_controller/config_schemas/deployment_updater_schema'
 require 'cloud_controller/config_schemas/rotate_database_key_schema'
+require 'cloud_controller/config_schemas/blobstore_benchmarks_schema'
 require 'utils/hash_utils'
 
 module VCAP::CloudController

lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb

@@ -0,0 +1,82 @@
+require 'vcap/config'
+
+module VCAP::CloudController
+  module ConfigSchemas
+    class BlobstoreBenchmarksSchema < VCAP::Config
+      # rubocop:disable Metrics/BlockLength
+      define_schema do
+        blobstore_section = {
+          optional(:blobstore_type) => String,
+          optional(:blobstore_provider) => String,
+
+          optional(:fog_connection) => Hash,
+          optional(:connection_config) => Hash,
+          optional(:webdav_config) => Hash,
+
+          optional(:fog_aws_storage_options) => Hash,
+          optional(:fog_gcp_storage_options) => Hash,
+
+          optional(:resource_directory_key) => String,
+          optional(:buildpack_directory_key) => String,
+          optional(:app_package_directory_key) => String,
+          optional(:droplet_directory_key) => String,
+
+          optional(:maximum_size) => Integer,
+          optional(:minimum_size) => Integer,
+          optional(:max_package_size) => Integer,
+          optional(:max_valid_packages_stored) => Integer,
+          optional(:max_staged_droplets_stored) => Integer
+        }
+
+        {
+          optional(:logging) => {
+            optional(:level) => String,
+            optional(:file) => String,
+            optional(:syslog) => String,
+            optional(:stdout_sink_enabled) => bool
+          },
+
+          db: {
+            optional(:database) => Hash,
+            optional(:db_connection_string) => String,
+            optional(:max_connections) => Integer,
+            optional(:pool_timeout) => Integer,
+            optional(:log_level) => String,
+            optional(:log_db_queries) => bool,
+            optional(:ssl_verify_hostname) => bool,
+            optional(:connection_validation_timeout) => Integer,
+            optional(:ca_cert_path) => String
+          },
+          optional(:storage_cli_config_file_resource_pool) => String,
+          optional(:storage_cli_config_file_buildpacks) => String,
+          optional(:storage_cli_config_file_packages) => String,
+          optional(:storage_cli_config_file_droplets) => String,
+
+          optional(:db_encryption_key) => enum(String, NilClass),
+          optional(:database_encryption) => {
+            keys: Hash,
+            current_key_label: String,
+            optional(:pbkdf2_hmac_iterations) => Integer
+          },
+
+          resource_pool: blobstore_section,
+          buildpacks: blobstore_section,
+          packages: blobstore_section,
+          droplets: blobstore_section,
+
+          optional(:pid_filename) => String,
+          optional(:index) => Integer,
+          optional(:name) => String,
+          optional(:default_app_ssh_access) => bool
+        }
+      end
+      # rubocop:enable Metrics/BlockLength
+
+      class << self
+        def configure_components(config)
+          ResourcePool.instance = ResourcePool.new(config)
+        end
+      end
+    end
+  end
+end

lib/cloud_controller/dependency_locator.rb

@@ -131,9 +131,8 @@ def index_stopper
       @dependencies[:index_stopper] || register(:index_stopper, IndexStopper.new(runners))
     end
 
-    def droplet_blobstore(blobstore_type: nil)
-      options = config.get(:droplets).dup
-      options[:blobstore_type] = blobstore_type if blobstore_type
+    def droplet_blobstore
+      options = config.get(:droplets)
 
       Blobstore::ClientProvider.provide(
         options: options,
@@ -153,9 +152,8 @@ def buildpack_cache_blobstore
       )
     end
 
-    def package_blobstore(blobstore_type: nil)
-      options = config.get(:packages).dup
-      options[:blobstore_type] = blobstore_type if blobstore_type
+    def package_blobstore
+      options = config.get(:packages)
 
       Blobstore::ClientProvider.provide(
         options: options,
@@ -185,9 +183,8 @@ def global_app_bits_cache
       )
     end
 
-    def buildpack_blobstore(blobstore_type: nil)
-      options = config.get(:buildpacks).dup
-      options[:blobstore_type] = blobstore_type if blobstore_type
+    def buildpack_blobstore
+      options = config.get(:buildpacks)
 
       Blobstore::ClientProvider.provide(
         options: options,

lib/tasks/blobstore_benchmarks.rake

@@ -3,7 +3,8 @@ require 'cloud_controller/benchmark/blobstore'
 namespace :benchmarks do
   desc 'Perform blobstore benchmark'
   task perform_blobstore_benchmark: :environment do
-    BackgroundJobEnvironment.new(RakeConfig.config).setup_environment do
+    RakeConfig.context = :blobstore_benchmarks
+    BoshErrandEnvironment.new(RakeConfig.config).setup_environment do
       VCAP::CloudController::Benchmark::Blobstore.new.perform
     end
   end