Record ETag and Last-Modified for each URL

Author: DilumAluthge

src/VersionsJSONUtil.jl

@@ -143,6 +143,38 @@ function get_tags()
     JSON.parse(String(read(tags_json_path)))
 end
 
+##### --------------------------------------------------------------------------------------
+##### Get ETag and Last-Modified, so we know if we need to re-download and re-checksum files
+
+Base.@kwdef struct HeadInfo
+    etag::Union{String, Nothing}
+    last_modified::Union{String, Nothing}
+end
+
+function get_new_headinfo_for_url(url::URI)
+    local response = nothing
+    try
+        response = HTTP.head(url)
+    catch
+        error("Encountered error when making HEAD request to URL: $url")
+    end
+    headers = Dict(HTTP.headers(response))
+    etag = get(headers, "ETag", "")
+    last_modified = get(headers, "Last-Modified", "")
+    if isempty(strip(etag))
+        @warn "Got empty new ETag for URL: $new_etag"
+        etag = nothing
+    end
+    if isempty(strip(last_modified))
+        @warn "Got empty new ETag for URL: $new_etag"
+        last_modified = nothing
+    end
+    head_info = HeadInfo(; etag, last_modified)
+    return head_info
+end
+
+##### --------------------------------------------------------------------------------------
+
 function main(out_path)
     tags = get_tags()
     tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags])
@@ -240,6 +272,8 @@ function main(out_path)
 
             end
 
+            headinfo = get_new_headinfo_for_url(url)
+
             # Build up metadata about this file
             file_dict = Dict(
                 "triplet" => triplet(platform),
@@ -260,6 +294,13 @@ function main(out_path)
                 file_dict["asc"] = asc_signature
             end
 
+            if !isnothing(headinfo.etag)
+                file_dict["etag"] = headinfo.etag
+            end
+            if !isnothing(headinfo.last_modified)
+                file_dict["last-modified"] = headinfo.last_modified
+            end
+
             # Right now, all we have are archives, but let's be forward-thinking
             # and make this an array of dictionaries that is easy to extensibly match
             push!(meta[version]["files"], file_dict)

test/more_tests.jl

@@ -67,8 +67,10 @@ end
                     ]
                     optional_keys = [
                         "asc",
+                        "etag",
                         "git-tree-sha1",
                         "git-tree-sha256",
+                        "last-modified",
                     ]
                     allowed_keys = union(required_keys, optional_keys)
                     @test required_keys ⊆ collect(keys(filedict))