Merge branch 'main' into feature/runtime-node-topic

Author: kenyonj

Gemfile

@@ -3,7 +3,7 @@ source "https://rubygems.org"
 gem "faraday", "2.14.1"
 gem "faraday-retry", "2.4.0"
 gem "github-pages", "~> 232", group: :jekyll_plugins
-gem "json", "2.18.0"
+gem "json", "2.19.2"
 gem "language_server-protocol", "3.17.0.5"
 gem "nokogiri", "~> 1.19.0"
 gem "rake", "13.3.1"

Gemfile.lock

@@ -236,7 +236,7 @@ GEM
       gemoji (>= 3, < 5)
       html-pipeline (~> 2.2)
       jekyll (>= 3.0, < 5.0)
-    json (2.18.0)
+    json (2.19.2)
     kramdown (2.4.0)
       rexml
     kramdown-parser-gfm (1.1.0)
@@ -371,7 +371,7 @@ DEPENDENCIES
   fastimage
   github-pages (~> 232)
   httparty
-  json (= 2.18.0)
+  json (= 2.19.2)
   language_server-protocol (= 3.17.0.5)
   minitest
   nokogiri (~> 1.19.0)

collections/machine-learning/index.md

@@ -39,7 +39,6 @@ items:
  - https://www.youtube.com/embed/bHvf7Tagt18
  - https://www.youtube.com/embed/ILsA4nyG7I0
  - AMAI-GmbH/AI-Expert-Roadmap
- - vitali87/code-graph-rag
 display_name: Getting started with machine learning
 created_by: omoju
 ---

collections/software-development-tools/index.md

@@ -18,7 +18,6 @@ items:
  - AhmedOsman101/commit-sage-cli
  - semaphoreio/semaphore
  - Nayjest/Gito
- - vitali87/code-graph-rag
 display_name: Software development tools
 ---
 Build apps better, faster, stronger.

collections/voxel-editors/index.md

@@ -6,7 +6,6 @@ items:
  - grking/zoxel
  - Perkovec/Vuxel
  - GaidamakUA/Voxenko
- - gerardparareda/BlockyTracer
  - Bequen/GridEditor
  - honestabelink/stonevox3d
  - jval1972/DD_VOXEL

test/collections_test.rb

@@ -140,6 +140,7 @@
         if ENV["SKIP_COLLECTION_API_CHECKS"]
           skip "Skipping collection API checks (rename detection handled by collections-renames)"
         end
+        prefetch_all_collection_items!
 
         errors = []
         repos_to_check = []
@@ -155,10 +156,6 @@
           end
         end
 
-        cache_repos_exist_check!(repos_to_check)
-        cache_users_exist_check!(users_to_check)
-        cache_orgs_exist_check!(users_not_found_from(users_to_check))
-
         repos_to_check.each do |repo|
           repo_result = client.repository(repo)
           current_name_with_owner = repo_result&.full_name

test/collections_test_helper.rb

@@ -118,3 +118,42 @@ def annotate_collection_item_error(collection, string, error_message)
 def possible_image_file_names_for_collection(collection)
   COLLECTION_IMAGE_EXTENSIONS.map { |ext| "#{collection}#{ext}" }
 end
+
+GRAPHQL_BATCH_SIZE = 100
+
+def prefetch_all_collection_items!
+  return if NewOctokit.global_prefetch_done?
+
+  repos, users = collect_all_collection_items
+  prefetch_repos!(repos)
+  prefetch_users!(users)
+
+  NewOctokit.global_prefetch_done!
+end
+
+def collect_all_collection_items
+  all_items = collections.flat_map { |c| items_for_collection(c) || [] }
+
+  repos = all_items.select { |item| item.match?(USERNAME_AND_REPO_REGEX) }.uniq
+  users = all_items
+          .select { |item| item.match?(USERNAME_REGEX) && !item.match?(USERNAME_AND_REPO_REGEX) }
+          .uniq
+
+  [repos, users]
+end
+
+def prefetch_repos!(repos)
+  repos.each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_repos_exist_check!(batch)
+  end
+end
+
+def prefetch_users!(users)
+  users.each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_users_exist_check!(batch)
+  end
+
+  users_not_found_from(users).each_slice(GRAPHQL_BATCH_SIZE) do |batch|
+    cache_orgs_exist_check!(batch)
+  end
+end

test/test_helper.rb

@@ -59,6 +59,7 @@ class NewOctokit < Octokit::Client
   @@repo_request_count = 0 unless defined? @@repo_request_count
   @@user_request_count = 0 unless defined? @@user_request_count
   @@messages = [] unless defined? @@messages
+  @@global_prefetch_done = false unless defined? @@global_prefetch_done
 
   def repos
     @@repos
@@ -122,6 +123,14 @@ def self.messages
     @@messages
   end
 
+  def self.global_prefetch_done?
+    @@global_prefetch_done
+  end
+
+  def self.global_prefetch_done!
+    @@global_prefetch_done = true
+  end
+
   # rubocop:enable Style/ClassVars
 end