Reduce peak memory usage of full index loading and bundle install

bundler/lib/bundler/definition.rb

@@ -236,6 +236,16 @@ def prefer_local!
       sources.prefer_local!
     end
 
+    # Releases memory only needed during resolution, such as remote spec
+    # indexes and resolver state. Only safe to call once resolution is
+    # complete and the result has been materialized, since any further
+    # resolution will need to refetch remote specs.
+    def release_resolution_memory!
+      @resolver = nil
+      @resolution_base = nil
+      sources.release_resolution_memory!
+    end
+
     # For given dependency list returns a SpecSet with Gemspec of all the required
     # dependencies.
     #  1. The method first resolves the dependencies specified in Gemfile

bundler/lib/bundler/fetcher.rb

@@ -243,6 +243,10 @@ def api_fetcher?
       fetchers.first.api_fetcher?
     end
 
+    def release_resolution_memory!
+      @fetchers&.each(&:release_resolution_memory!)
+    end
+
     def gem_remote_fetcher
       @gem_remote_fetcher ||= begin
         require_relative "fetcher/gem_remote_fetcher"

bundler/lib/bundler/fetcher/base.rb

@@ -38,6 +38,9 @@ def api_fetcher?
         false
       end
 
+      def release_resolution_memory!
+      end
+
       private
 
       def log_specs(&block)

bundler/lib/bundler/fetcher/compact_index.rb

@@ -63,6 +63,13 @@ def api_fetcher?
         true
       end
 
+      # The client holds the parsed checksums of all info files in the
+      # index. Dropping it is always safe because it is rebuilt from the
+      # local cache on demand.
+      def release_resolution_memory!
+        @compact_index_client = nil
+      end
+
       private
 
       def compact_index_client
@@ -73,6 +80,12 @@ def compact_index_client
       end
 
       def fetch_gem_infos(names)
+        # Create the client and update the versions file on this thread.
+        # Otherwise the workers race to lazily create the client and update
+        # the versions file concurrently, e.g. when the client was released
+        # after resolution and is being rebuilt for `bundle cache`.
+        compact_index_client.available?
+
         in_parallel(names) {|name| compact_index_client.info(name) }
       rescue TooManyRequestsError # rubygems.org is rate limiting us, slow down.
         @bundle_worker&.stop

bundler/lib/bundler/installer.rb

@@ -195,7 +195,13 @@ def install(options)
       force = options[:force]
       local = options[:local] || options[:"prefer-local"]
       jobs = Bundler.settings.installation_parallelization
-      spec_installations = ParallelInstaller.call(self, @definition.specs, jobs, standalone, force, local: local)
+      specs = @definition.specs
+      # Installing default gems may need the remote index again to cache
+      # their .gem files, so keep resolution memory around in that case.
+      # The bundler spec itself is excluded because it comes from the
+      # metadata source and never goes through that path.
+      @definition.release_resolution_memory! if specs.none? {|s| s.default_gem? && s.source.is_a?(Source::Rubygems) }
+      spec_installations = ParallelInstaller.call(self, specs, jobs, standalone, force, local: local)
       spec_installations.each do |installation|
         post_install_messages[installation.name] = installation.post_install_message if installation.has_post_install_message?
       end

bundler/lib/bundler/source/rubygems.rb

@@ -25,6 +25,7 @@ def initialize(options = {})
         @checksum_store = Checksum::Store.new
         @gem_installers = {}
         @gem_installers_mutex = Mutex.new
+        @remote_specs_mutex = Mutex.new
 
         cooldown = options["cooldown"]
         Array(options["remotes"]).reverse_each {|r| add_remote(r, cooldown: cooldown) }
@@ -243,7 +244,7 @@ def cache(spec, custom_path = nil)
       def cached_built_in_gem(spec, local: false)
         cached_path = cached_gem(spec)
         if cached_path.nil? && !local
-          remote_spec = remote_specs.search(spec).first
+          remote_spec = remote_spec_for(spec)
           if remote_spec
             cached_path = fetch_gem(remote_spec)
             spec.remote = remote_spec.remote
@@ -337,6 +338,12 @@ def clear_cache
         @cached_specs = nil
       end
 
+      def release_resolution_memory!
+        @specs = nil
+        @remote_specs_mutex.synchronize { @remote_specs = nil }
+        @fetchers&.each(&:release_resolution_memory!)
+      end
+
       protected
 
       def remote_names
@@ -414,17 +421,30 @@ def api_fetchers
       end
 
       def remote_specs
-        @remote_specs ||= Index.build do |idx|
-          index_fetchers = fetchers - api_fetchers
+        @remote_specs ||= @remote_specs_mutex.synchronize do
+          @remote_specs ||= Index.build do |idx|
+            index_fetchers = fetchers - api_fetchers
 
-          if index_fetchers.empty?
-            fetch_names(api_fetchers, dependency_names, idx)
-          else
-            fetch_names(fetchers, nil, idx)
+            if index_fetchers.empty?
+              fetch_names(api_fetchers, dependency_names, idx)
+            else
+              fetch_names(fetchers, nil, idx)
+            end
           end
         end
       end
 
+      # Looks up a single spec in the remote sources, fetching only its own
+      # name when the full remote index is not already materialized.
+      def remote_spec_for(spec)
+        return remote_specs.search(spec).first if @remote_specs || api_fetchers.empty?
+
+        index = Index.build do |idx|
+          fetch_names(api_fetchers, [spec.name], idx)
+        end
+        index.search(spec).first
+      end
+
       def fetch_names(fetchers, dependency_names, index)
         fetchers.each do |f|
           if dependency_names

bundler/lib/bundler/source_list.rb

@@ -140,6 +140,10 @@ def clear_cache
       rubygems_sources.each(&:clear_cache)
     end
 
+    def release_resolution_memory!
+      rubygems_sources.each(&:release_resolution_memory!)
+    end
+
     private
 
     def map_sources(replacement_sources)

lib/rubygems/safe_marshal/reader.rb

@@ -28,6 +28,8 @@ class NegativeLengthError < Error
 
       def initialize(io)
         @io = io
+        @object_links = {}
+        @symbol_links = {}
       end
 
       def read!
@@ -191,7 +193,7 @@ def read_object_with_ivars
 
       def read_symbol_link
         offset = read_integer
-        Elements::SymbolLink.new(offset)
+        @symbol_links[offset] ||= Elements::SymbolLink.new(offset)
       end
 
       def read_user_marshal
@@ -200,43 +202,9 @@ def read_user_marshal
         Elements::UserMarshal.new(name, data)
       end
 
-      # profiling bundle install --full-index shows that
-      # offset 6 is by far the most common object link,
-      # so we special case it to avoid allocating a new
-      # object a third of the time.
-      # the following are all the object links that
-      # appear more than 10000 times in my profiling
-
-      OBJECT_LINKS = {
-        6 => Elements::ObjectLink.new(6).freeze,
-        30 => Elements::ObjectLink.new(30).freeze,
-        81 => Elements::ObjectLink.new(81).freeze,
-        34 => Elements::ObjectLink.new(34).freeze,
-        38 => Elements::ObjectLink.new(38).freeze,
-        50 => Elements::ObjectLink.new(50).freeze,
-        91 => Elements::ObjectLink.new(91).freeze,
-        42 => Elements::ObjectLink.new(42).freeze,
-        46 => Elements::ObjectLink.new(46).freeze,
-        150 => Elements::ObjectLink.new(150).freeze,
-        100 => Elements::ObjectLink.new(100).freeze,
-        104 => Elements::ObjectLink.new(104).freeze,
-        108 => Elements::ObjectLink.new(108).freeze,
-        242 => Elements::ObjectLink.new(242).freeze,
-        246 => Elements::ObjectLink.new(246).freeze,
-        139 => Elements::ObjectLink.new(139).freeze,
-        143 => Elements::ObjectLink.new(143).freeze,
-        114 => Elements::ObjectLink.new(114).freeze,
-        308 => Elements::ObjectLink.new(308).freeze,
-        200 => Elements::ObjectLink.new(200).freeze,
-        54 => Elements::ObjectLink.new(54).freeze,
-        62 => Elements::ObjectLink.new(62).freeze,
-        1_286_245 => Elements::ObjectLink.new(1_286_245).freeze,
-      }.freeze
-      private_constant :OBJECT_LINKS
-
       def read_object_link
         offset = read_integer
-        OBJECT_LINKS[offset] || Elements::ObjectLink.new(offset)
+        @object_links[offset] ||= Elements::ObjectLink.new(offset)
       end
 
       EMPTY_HASH = Elements::Hash.new([].freeze).freeze