Record ETag and Last-Modified for each URL

Author: DilumAluthge

src/VersionsJSONUtil.jl

@@ -132,6 +132,38 @@ function get_tags()
     JSON.parse(String(read(tags_json_path)))
 end
 
+##### --------------------------------------------------------------------------------------
+##### Get ETag and Last-Modified, so we know if we need to re-download and re-checksum files
+
+Base.@kwdef struct HeadInfo
+    etag::Union{String, Nothing}
+    last_modified::Union{String, Nothing}
+end
+
+function get_new_headinfo_for_url(url::URI)
+    local response = nothing
+    try
+        response = HTTP.head(url)
+    catch
+        error("Encountered error when making HEAD request to URL: $url")
+    end
+    headers = Dict(HTTP.headers(response))
+    etag = get(headers, "ETag", "")
+    last_modified = get(headers, "Last-Modified", "")
+    if isempty(strip(etag))
+        @warn "Got empty new ETag for URL: $new_etag"
+        etag = nothing
+    end
+    if isempty(strip(last_modified))
+        @warn "Got empty new ETag for URL: $new_etag"
+        last_modified = nothing
+    end
+    head_info = HeadInfo(; etag, last_modified)
+    return head_info
+end
+
+##### --------------------------------------------------------------------------------------
+
 function main(out_path)
     tags = get_tags()
     tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags])
@@ -211,6 +243,9 @@ function main(out_path)
             else
                 error("Unsupported file extension in filename: $(filename)")
             end
+
+            headinfo = get_new_headinfo_for_url(url)
+            
             file_dict = Dict(
                 "triplet" => triplet(platform),
                 "os" => meta_os(platform),
@@ -227,6 +262,13 @@ function main(out_path)
                 file_dict["asc"] = asc_signature
             end
 
+            if !isnothing(headinfo.etag)
+                file_dict["etag"] = headinfo.etag
+            end
+            if !isnothing(headinfo.last_modified)
+                file_dict["last-modified"] = headinfo.last_modified
+            end
+            
             # Right now, all we have are archives, but let's be forward-thinking
             # and make this an array of dictionaries that is easy to extensibly match
             push!(meta[version]["files"], file_dict)