stuff

Author: DilumAluthge

.github/workflows/CI.yml

@@ -65,13 +65,16 @@ jobs:
       - name: Install dependencies
         run: julia --color=yes --project -e "using Pkg; Pkg.instantiate()"
       - name: Cleanup before we build
-        run: make nuke
+        run: |
+          make clean
+          make nuke
+          make purge_download_cache_dir
       - name: Download the old versions.json, so we can re-use it
         run: curl -LO https://julialang-s3.julialang.org/bin/versions.json
       - name: Build versions.json
         run: make build
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Used to authenticate to GitHub API to get list of tags
       - name: Run the post-build tests
         run: make test-full
       - name: Install NPM deps

.gitignore

@@ -1,4 +1,5 @@
 # Directories:
+cache/
 node_modules/
 
 # Files:

Makefile

@@ -54,3 +54,7 @@ nuke:
 .PHONY: _purge_node_modules
 _purge_node_modules:
 	$(RM) -r test/node/node_modules/
+
+.PHONY: purge_download_cache_dir
+purge_download_cache_dir:
+	$(RM) -r cache/

src/VersionsJSONUtil.jl

@@ -3,6 +3,7 @@ module VersionsJSONUtil
 ##### --------------------------------------------------------------------------------------
 ##### Import dependencies
 
+using Downloads: Downloads
 using EnumX: EnumX, @enumx
 using GitHub: GitHub
 using HTTP: HTTP
@@ -26,7 +27,7 @@ include("sha_structs.jl")
 
 include("typed_json.jl")
 
-include("download.jl")
+include("web_cache_utilities.jl")
 
 ##### --------------------------------------------------------------------------------------
 ##### Config, OutputJsonContent
@@ -331,16 +332,18 @@ function filedict_is_complete_and_good(filedict::FileDict)
 
     # Now the (compared to the rest of this function) expensive part: Check our Etag.
     url = filedict.url
-    old_headinfo = HeadInfo(;
-        etag = filedict.etag,
-        last_modified = filedict.last_modified,
-    )
+    old_etag = filedict.etag
+    old_last_modified = filedict.last_modified
     new_headinfo = get_new_headinfo_for_url(url)
     if isnothing(new_headinfo)
         return false
     end
-    if old_headinfo != new_headinfo
-        @warn "HeadInfo for URL $url has changed from $old_headinfo to $new_headinfo"
+    if old_etag != new_headinfo.etag
+        @warn "ETag for URL $url has changed from $(old_etag) to $(new_headinfo.etag)"
+        return false
+    end
+    if old_last_modified != new_headinfo.last_modified
+        @warn "Last-Modified for URL $url has changed from $(old_last_modified) to $(new_headinfo.last_modified)"
         return false
     end
 
@@ -433,9 +436,13 @@ function get_tags()
     else
         # The user didn't provide GITHUB_TOKEN, so we have to make an unauthenticated call
         # to the GitHub API.
-        # Warning: The rate limit for unauthenticated calls is very low.
+        # The rate limit for unauthenticated calls is very low, so we cache this.
         @warn "GITHUB_TOKEN not detected. It is recommended that you provide a GITHUB_TOKEN (with read-only access to only public repos)."
-        return GitHub.gh_get_json(GitHub.DEFAULT_API, "/$request_path"; auth)
+        tags_json_path = download_to_cache(
+            "julia_tags.json",
+            "https://api.github.com/$request_path",
+        )
+        return JSON.parse(String(read(tags_json_path)))
     end
 end
 
@@ -540,7 +547,7 @@ function main!(content::OutputJsonContent, cfg::Config)
 
                 # Download this URL to a local file in a temp directory
                 number_urls_tried += 1
-                filepath = Downloads.download(url)
+                filepath = Downloads.download(string(url))
 
                 if filesize(filepath) == 0
                     # The file is empty
@@ -572,7 +579,7 @@ function main!(content::OutputJsonContent, cfg::Config)
                     kind = "archive"
                     extension = "zip"
                 else
-                    error("Unsupported file extension in filename: $(filename)")
+                    error("Unsupported file extension in filename: filename")
                 end
 
                 if extension == "tar.gz" && !(url in tarball_git_tree_hash_skiplist)

src/download.jl

@@ -0,0 +1,59 @@
+using Dates: Dates, now, UTC, unix2datetime, TimePeriod, Hour
+using URIs: URI
+
+function download_to_cache(filename::String, url::Union{String, URI})
+    file_path = _hit_file_cache(filename) do file_path
+        r = open(io -> HTTP.get(url; response_stream=io), file_path, "w")
+        if r.status != 200
+            error("Unable to download $(url) to $(filename)")
+        end
+    end
+    return file_path
+end
+
+function _hit_file_cache(creator::Function, filename::String, lifetime::TimePeriod = Hour(24))
+    cache_dir = abspath(joinpath(@__DIR__, "..", "cache"))
+    if !isdir(cache_dir)
+        mkpath(cache_dir)
+    end
+
+    cache_path = joinpath(cache_dir, filename)
+    if isfile(cache_path)
+        age = now(UTC) - unix2datetime(stat(cache_path).mtime)
+        if age < lifetime
+            @debug "Cache hit" filename isfile(cache_path) age Dates.canonicalize(age) lifetime
+            return abspath(cache_path)
+        else
+            @debug "Cache miss. Cache entry found, but stale (too old)" filename isfile(cache_path) age Dates.canonicalize(age) lifetime
+        end
+    else
+        @debug "Cache miss. Cache entry not found" filename isfile(cache_path)
+    end
+
+    rm(cache_path; force=true)
+    mktempdir() do tmpdir
+        tmp_path = joinpath(tmpdir, filename)
+
+        creator(tmp_path)
+
+        if !isfile(tmp_path)
+            error("Creator function didn't actually create file: $filename")
+        end
+        try
+            mv(tmp_path, cache_path; force = true)
+        catch
+            rm(cache_path; force=true)
+            rethrow()
+        end
+    end
+
+    if !isfile(cache_path)
+        error("Our mv must have somehow failed for file: $filename")
+    end
+    # Update the mtime:
+    touch(cache_path)
+
+    @debug "Successfully saved new cache entry" filename
+
+    return abspath(cache_path)
+end