Merge pull request #5043 from github/global-graphql-batch-prefetch

Prefetch all collection items globally before tests

Author: kenyonj

test/collections_test.rb

@@ -140,6 +140,7 @@
         if ENV["SKIP_COLLECTION_API_CHECKS"]
           skip "Skipping collection API checks (rename detection handled by collections-renames)"
         end
+        prefetch_all_collection_items!
 
         errors = []
         repos_to_check = []
@@ -155,10 +156,6 @@
           end
         end
 
-        cache_repos_exist_check!(repos_to_check)
-        cache_users_exist_check!(users_to_check)
-        cache_orgs_exist_check!(users_not_found_from(users_to_check))
-
         repos_to_check.each do |repo|
           repo_result = client.repository(repo)
           current_name_with_owner = repo_result&.full_name

test/collections_test_helper.rb

@@ -118,3 +118,42 @@ def annotate_collection_item_error(collection, string, error_message)
 def possible_image_file_names_for_collection(collection)
   COLLECTION_IMAGE_EXTENSIONS.map { |ext| "#{collection}#{ext}" }
 end
+
+GRAPHQL_BATCH_SIZE = 100
+
+def prefetch_all_collection_items!
+  return if NewOctokit.global_prefetch_done?
+
+  repos, users = collect_all_collection_items
+  prefetch_repos!(repos)
+  prefetch_users!(users)
+
+  NewOctokit.global_prefetch_done!
+end
+
+def collect_all_collection_items
+  all_items = collections.flat_map { |c| items_for_collection(c) || [] }
+
+  repos = all_items.select { |item| item.match?(USERNAME_AND_REPO_REGEX) }.uniq
+  users = all_items
+          .select { |item| item.match?(USERNAME_REGEX) && !item.match?(USERNAME_AND_REPO_REGEX) }
+          .uniq
+
+  [repos, users]
+end
+
+def prefetch_repos!(repos)
+  repos.each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_repos_exist_check!(batch)
+  end
+end
+
+def prefetch_users!(users)
+  users.each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_users_exist_check!(batch)
+  end
+
+  users_not_found_from(users).each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_orgs_exist_check!(batch)
+  end
+end

test/test_helper.rb

@@ -59,6 +59,7 @@ class NewOctokit < Octokit::Client
   @@repo_request_count = 0 unless defined? @@repo_request_count
   @@user_request_count = 0 unless defined? @@user_request_count
   @@messages = [] unless defined? @@messages
+  @@global_prefetch_done = false unless defined? @@global_prefetch_done
 
   def repos
     @@repos
@@ -122,6 +123,14 @@ def self.messages
     @@messages
   end
 
+  def self.global_prefetch_done?
+    @@global_prefetch_done
+  end
+
+  def self.global_prefetch_done!
+    @@global_prefetch_done = true
+  end
+
   # rubocop:enable Style/ClassVars
 end