my_sleep()

Author: DilumAluthge

src/VersionsJSONUtil.jl

@@ -359,7 +359,17 @@ Base.@kwdef struct HeadInfo
     last_modified::Union{String, Nothing}
 end
 
+# We call this function before hitting an upstream server.
+# By default, it is a sleep(0), but we can make this user-configurable.
+# So that way, if a user is using a mirror that has rate-limits, they can configure this
+# to help them stay under the limits.
+function my_sleep()
+    sleep(0)
+    return nothing
+end
+
 function get_new_headinfo_for_url(url::URI)
+    my_sleep()
     response = HTTP.head(url; status_exception = false)
     status = response.status
     if status != 200
@@ -428,10 +438,16 @@ function get_tags()
     @info("Probing for tag list...")
     request_path = "repos/JuliaLang/julia/git/refs/tags"
     token = strip(get(ENV, "GITHUB_TOKEN", ""))
+
+
+
     if !isempty(token)
         # The user provided GITHUB_TOKEN, so we use that to authenticate to GitHub API.
         # We don't cache this result - this ensures we get the freshest data.
         auth = GitHub.authenticate(token)
+
+        # Note: GitHub.jl now has rate-limit support built-in.
+        # So we don't call my_sleep() here.
         return GitHub.gh_get_json(GitHub.DEFAULT_API, "/$request_path"; auth)
     else
         # The user didn't provide GITHUB_TOKEN, so we have to make an unauthenticated call
@@ -549,6 +565,7 @@ function main!(content::OutputJsonContent, cfg::Config)
                 number_urls_tried += 1
 
                 # This will download the file to a tempfile
+                my_sleep()
                 filepath = Downloads.download(string(url))
 
                 if filesize(filepath) == 0
@@ -643,6 +660,7 @@ function main!(content::OutputJsonContent, cfg::Config)
                     asc_filename = basename(string(asc_url))
                     @debug "Downloading $asc_filename"
                     try
+                        my_sleep()
                         asc_filepath = Downloads.download(string(asc_url))
                         asc_signature = String(read(asc_filepath))
                         rm(asc_filepath)

src/web_cache_utilities.jl

@@ -2,6 +2,7 @@ using Dates: Dates, now, UTC, unix2datetime, TimePeriod, Hour
 using URIs: URI
 
 function download_to_cache(filename::String, url::Union{String, URI})
+    my_sleep()
     file_path = _hit_file_cache(filename) do file_path
         r = open(io -> HTTP.get(url; response_stream=io), file_path, "w")
         if r.status != 200