add more bigger files for upload time measuring

Author: kathap

lib/cloud_controller/benchmark/blobstore.rb

@@ -1,77 +1,78 @@
+# frozen_string_literal: true
+
 require 'benchmark'
 require 'find'
 require 'zip'
+require 'tempfile'
+require 'fileutils'
+require 'securerandom'
 
 module VCAP::CloudController
   module Benchmark
     class Blobstore
+      SIZES = [
+        ['0.005MB', (0.005 * 1024 * 1024).to_i],
+        ['0.01MB',  (0.01  * 1024 * 1024).to_i],
+        ['0.1MB',   (0.1   * 1024 * 1024).to_i],
+        ['1MB',     1    * 1024 * 1024],
+        ['10MB',    10   * 1024 * 1024],
+        ['100MB',   100  * 1024 * 1024],
+        ['200MB',   200  * 1024 * 1024],
+        ['300MB',   300  * 1024 * 1024],
+        ['400MB',   400  * 1024 * 1024],
+        ['500MB',   500  * 1024 * 1024],
+        ['600MB',   600  * 1024 * 1024],
+        ['700MB',   700  * 1024 * 1024],
+        ['800MB',   800  * 1024 * 1024],
+        ['900MB',   900  * 1024 * 1024],
+        ['1000MB',  1000 * 1024 * 1024]
+      ].freeze
+
+      CHUNK_1MB = '0'.b * (1024 * 1024)
+
       def perform
+        big_droplet_guids = []
         resource_dir = generate_resources
-
-        resource_timing = resource_match(resource_dir)
-        puts("resource match timing: #{resource_timing * 1000}ms")
+        log_timing('resource match timing', resource_match(resource_dir))
 
         zip_output_dir = Dir.mktmpdir
         zip_file = zip_resources(resource_dir, zip_output_dir)
 
-        package_guid, resource_timing = upload_package(zip_file)
-        puts("package upload timing: #{resource_timing * 1000}ms")
-
-        resource_timing = download_package(package_guid, resource_dir)
-        puts("package download timing: #{resource_timing * 1000}ms")
+        package_guid, timing = upload_package(zip_file)
+        log_timing('package upload timing', timing)
+        log_timing('package download timing', download_package(package_guid, resource_dir))
 
-        bytes_read, resource_timing = download_buildpacks(resource_dir)
+        bytes_read, timing = download_buildpacks(resource_dir)
         puts("downloaded #{Buildpack.count} buildpacks, total #{bytes_read} bytes read")
-        puts("buildpack download timing: #{resource_timing * 1000}ms")
-
-        droplet_guid, resource_timing = upload_droplet(zip_file)
-        puts("droplet upload timing: #{resource_timing * 1000}ms")
-
-        resource_timing = download_droplet(droplet_guid, resource_dir)
-        puts("droplet download timing: #{resource_timing * 1000}ms")
-
-        big_droplet_file = Tempfile.new('big-droplet', resource_dir)
-        big_droplet_file.write('abc' * 1024 * 1024 * 100)
-        big_droplet_guid, resource_timing = upload_droplet(big_droplet_file.path)
-        puts("big droplet upload timing: #{resource_timing * 1000}ms")
-
-        [
-          ['0.005MB', (0.005 * 1024 * 1024).to_i],
-          ['10MB', 10 * 1024 * 1024],
-          ['200MB', 200 * 1024 * 1024],
-          ['500MB', 500 * 1024 * 1024]
-        ].each do |label, size|
-          tempfile = Tempfile.new("big-droplet-#{label}", resource_dir)
-          File.open(tempfile.path, 'wb') do |f|
-            chunk = '0' * (1024 * 1024) # 1MB chunk
-            written = 0
-            while written < size
-              to_write = [chunk.bytesize, size - written].min
-              f.write(chunk.byteslice(0, to_write))
-              written += to_write
-            end
-          end
+        log_timing('buildpack download timing', timing)
 
-          big_droplet_guid, resource_timing = upload_droplet(tempfile.path)
-          puts("big droplet #{label} upload timing: #{resource_timing * 1000}ms")
+        droplet_guid, timing = upload_droplet(zip_file)
+        log_timing('droplet upload timing', timing)
+        log_timing('droplet download timing', download_droplet(droplet_guid, resource_dir))
 
-          tempfile.close!
-        end
+        SIZES.each do |label, bytes|
+          Tempfile.create(["big-droplet-#{label}", '.bin'], resource_dir) do |tempfile|
+            write_file_of_size(tempfile.path, bytes)
 
-        resource_timing = download_droplet(big_droplet_guid, resource_dir)
-        puts("big droplet download timing: #{resource_timing * 1000}ms")
+            guid, upload_timing = upload_droplet(tempfile.path)
+            big_droplet_guids << guid
+            log_timing("big droplet #{label} upload timing", upload_timing)
+            log_timing("big droplet #{label} download timing", download_droplet(guid, resource_dir))
+          end
+        end
       ensure
-        FileUtils.remove_dir(resource_dir, true)
-        FileUtils.remove_dir(zip_output_dir, true)
-        package_blobstore_client.delete(package_guid) if package_guid
-        droplet_blobstore_client.delete(droplet_guid) if droplet_guid
-        droplet_blobstore_client.delete(big_droplet_guid) if big_droplet_guid
+        FileUtils.remove_dir(resource_dir, true) if resource_dir
+        FileUtils.remove_dir(zip_output_dir, true) if zip_output_dir
+
+        safe_delete(package_blobstore_client, package_guid)
+        safe_delete(droplet_blobstore_client, droplet_guid)
+        Array(big_droplet_guids).each { |g| safe_delete(droplet_blobstore_client, g) }
       end
 
       def resource_match(dir_path)
-        resources = Find.find(dir_path).
-                    select { |f| File.file?(f) }.
-                    map { |f| { 'size' => File.stat(f).size, 'sha1' => Digester.new.digest_path(f) } }
+        resources = Find.find(dir_path)
+                        .select { |f| File.file?(f) }
+                        .map { |f| { 'size' => File.stat(f).size, 'sha1' => Digester.new.digest_path(f) } }
 
         ::Benchmark.realtime do
           resource_pool.match_resources(resources)
@@ -83,46 +84,69 @@ def upload_package(package_path)
       end
 
       def download_package(package_guid, tmp_dir)
-        tempfile = Tempfile.new('package-download-benchmark', tmp_dir)
-        ::Benchmark.realtime do
-          package_blobstore_client.download_from_blobstore(package_guid, tempfile.path)
+        Tempfile.create('package-download-benchmark', tmp_dir) do |tempfile|
+          ::Benchmark.realtime do
+            package_blobstore_client.download_from_blobstore(package_guid, tempfile.path)
+          end
         end
       end
 
       def download_buildpacks(tmp_dir)
-        tempfile = Tempfile.new('buildpack-download-benchmark', tmp_dir)
-        bytes_read = 0
-
-        timing = ::Benchmark.realtime do
-          bytes_read = Buildpack.map do |buildpack|
-            buildpack_blobstore_client.download_from_blobstore(buildpack.key, tempfile.path)
-            File.stat(tempfile.path).size
-          end.sum
+        Tempfile.create('buildpack-download-benchmark', tmp_dir) do |tempfile|
+          bytes_read = 0
+          timing = ::Benchmark.realtime do
+            bytes_read = Buildpack.map do |buildpack|
+              buildpack_blobstore_client.download_from_blobstore(buildpack.key, tempfile.path)
+              File.stat(tempfile.path).size
+            end.sum
+          end
+          [bytes_read, timing]
         end
-
-        [bytes_read, timing]
       end
 
       def upload_droplet(droplet_path)
         copy_to_blobstore(droplet_path, droplet_blobstore_client)
       end
 
       def download_droplet(droplet_guid, tmp_dir)
-        tempfile = Tempfile.new('droplet-download-benchmark', tmp_dir)
-
-        ::Benchmark.realtime do
-          droplet_blobstore_client.download_from_blobstore(droplet_guid, tempfile.path)
+        Tempfile.create('droplet-download-benchmark', tmp_dir) do |tempfile|
+          ::Benchmark.realtime do
+            droplet_blobstore_client.download_from_blobstore(droplet_guid, tempfile.path)
+          end
         end
       end
 
       private
 
+      def log_timing(label, seconds)
+        puts("#{label}: #{(seconds * 1000).round(3)}ms")
+      end
+
+      def safe_delete(client, guid)
+        return if guid.nil?
+
+        client.delete(guid)
+      rescue StandardError => e
+        # don't fail the benchmark run if cleanup fails
+        warn("cleanup failed for guid=#{guid}: #{e.class}: #{e.message}")
+      end
+
+      def write_file_of_size(path, bytes)
+        File.open(path, 'wb') do |f|
+          remaining = bytes
+          while remaining > 0
+            to_write = [CHUNK_1MB.bytesize, remaining].min
+            f.write(CHUNK_1MB, to_write)
+            remaining -= to_write
+          end
+        end
+      end
+
       def generate_resources
         dir = Dir.mktmpdir
 
-        100.times.each do |i|
-          f = File.open(File.join(dir, i.to_s), 'w')
-          f.write('foo' * (65_536 + i))
+        100.times do |i|
+          File.write(File.join(dir, i.to_s), 'foo' * (65_536 + i))
         end
 
         dir
@@ -131,11 +155,9 @@ def generate_resources
       def zip_resources(resource_dir, output_dir)
         zip_file = File.join(output_dir, 'zipped_package')
         Zip::File.open(zip_file, create: true) do |zipfile|
-          Find.find(resource_dir).
-            select { |f| File.file?(f) }.
-            each do |file|
-            zipfile.add(File.basename(file), file)
-          end
+          Find.find(resource_dir)
+              .select { |f| File.file?(f) }
+              .each { |file| zipfile.add(File.basename(file), file) }
         end
         zip_file
       end