No need for TOML

Author: DilumAluthge

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.11"
 manifest_format = "2.0"
-project_hash = "11a43bcb5491dc8e3e69723068c666fcfe7c92d6"
+project_hash = "541a1e822f6feb6d841f99682e7d723700c0e5e2"
 
 [[deps.AbstractFFTs]]
 deps = ["LinearAlgebra"]

Project.toml

@@ -12,7 +12,6 @@ Lazy = "50d2b5c4-7a5e-59d5-8109-a42b560f39c0"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 WebCacheUtilities = "0c1c26de-fc5f-47ff-87a8-a157289a9bac"
 

Settings.toml

@@ -154,44 +154,50 @@ function get_tags()
     end
 end
 
-Base.@kwdef struct OutputDestination
+Base.@kwdef struct Config
     versions_json_filename::AbstractString
     internal_json_filename::AbstractString
 end
 
-function OutputDestination(output_directory::AbstractString)
+function Config(output_directory::AbstractString)
     versions_json_filename = joinpath(output_directory, "versions.json")
     internal_json_filename = joinpath(output_directory, "internal.json")
-    dest = OutputDestination(; versions_json_filename, internal_json_filename)
-    return dest
+
+    settings_file = joinpath(dirname(@__DIR__), "Settings.toml")
+    settings_toml = TOML.parsefile(settings_file)
+
+    cfg = Config(;
+        versions_json_filename,
+        internal_json_filename,
+    )
+    return cfg
 end
 
 struct OutputJsonContent
     versions_json::OrderedDict # this variable was previously named `meta`
     internal_json::OrderedDict
 end
 
-function OutputJsonContent(dest::OutputDestination)
-    if isfile(dest.versions_json_filename)
-        versions_json = OrderedDict(JSON.parsefile(dest.versions_json_filename))
+function OutputJsonContent(cfg::Config)
+    if isfile(cfg.versions_json_filename)
+        versions_json = OrderedDict(JSON.parsefile(cfg.versions_json_filename))
     else
         versions_json = OrderedDict()
     end
-    if isfile(dest.internal_json_filename)
-        internal_json = OrderedDict(JSON.parsefile(dest.internal_json_filename))
+    if isfile(cfg.internal_json_filename)
+        internal_json = OrderedDict(JSON.parsefile(cfg.internal_json_filename))
     else
         internal_json = OrderedDict()
     end
     return OutputJsonContent(versions_json, internal_json)
 end
 
-function checkpoint(content::OutputJsonContent, dest::OutputDestination)
-    indent = 2
-    open(dest.versions_json_filename, "w") do io
-        JSON.print(io, content.versions_json, indent)
+function checkpoint(content::OutputJsonContent, cfg::Config)
+    open(cfg.versions_json_filename, "w") do io
+        JSON.json(io, content.versions_json; pretty=true, sort_keys=false)
     end
-    open(dest.internal_json_filename, "w") do io
-        JSON.print(io, content.internal_json, indent)
+    open(cfg.internal_json_filename, "w") do io
+        JSON.json(io, content.internal_json; pretty=true, sort_keys=false)
     end
     return nothing
 end
@@ -341,7 +347,12 @@ function filedict_is_complete_and_good(filedict::AbstractDict)
 end
 
 function get_new_etag_for_url(url::AbstractString)::String
-    response = HTTP.head(url)
+    local response = nothing
+    try
+        response = HTTP.head(url)
+    catch
+        error("Encountered error when making HEAD request to URL: $url")
+    end
     new_etag = get(Dict(HTTP.headers(response)), "ETag", "")
     if isempty(strip(new_etag))
         @warn "Got empty new ETag for URL: $new_etag"
@@ -350,16 +361,6 @@ function get_new_etag_for_url(url::AbstractString)::String
     return new_etag
 end
 
-function main(output_directory::AbstractString)
-    dest = OutputDestination(output_directory)
-    return main(dest)
-end
-
-function main(dest::OutputDestination)
-    content = OutputJsonContent(dest)
-    return main!(content, dest)
-end
-
 function assert_sanity_check_tag_number(tag_versions::Vector{VersionNumber})
     # IIRC, we have never deleted a tag from JuliaLang/julia
     # So this number should be monotonically non-decreasing over time
@@ -374,7 +375,17 @@ function assert_sanity_check_tag_number(tag_versions::Vector{VersionNumber})
     end
 end
 
-function main!(content::OutputJsonContent, dest::OutputDestination)
+function main(output_directory::AbstractString)
+    cfg = Config(output_directory)
+    return main(cfg)
+end
+
+function main(cfg::Config)
+    content = OutputJsonContent(cfg)
+    return main!(content, cfg)
+end
+
+function main!(content::OutputJsonContent, cfg::Config)
     tags = get_tags()
     tag_versions = filter(x -> !isnothing(x), [VersionNumber(basename(t["ref"])) for t in tags])
     unique!(tag_versions)
@@ -394,6 +405,7 @@ function main!(content::OutputJsonContent, dest::OutputDestination)
     number_urls_success = 0
     num_urls_already_complete = 0
     number_urls_already_known_nonexistent = 0
+    checkpoint(content, cfg)
     for version in tag_versions
         for platform in julia_platforms
             url = download_url(version, platform)
@@ -421,14 +433,19 @@ function main!(content::OutputJsonContent, dest::OutputDestination)
                     if (ex isa HTTP.Exceptions.StatusError) && (ex.status == 404)
                         println(stdout, " ✗")
                         mark_url_as_nonexistent!(content, version, url)
-                        checkpoint(content, dest)
+                        checkpoint(content, cfg)
                         continue # skip the rest of this for-loop iteration
                     else
                         rethrow()
                     end
                 end
                 filepath::AbstractString
 
+                if filesize(filepath) == 0
+                    # The file is empty
+                    continue # skip the rest of this for-loop iteration
+                end
+
                 number_urls_success += 1
                 println(stdout, " ✓")
 
@@ -488,7 +505,7 @@ function main!(content::OutputJsonContent, dest::OutputDestination)
                 push!(meta[string(version)]["files"], file_dict)
 
                 # Write out new versions of our versions.json as we go
-                checkpoint(content, dest)
+                checkpoint(content, cfg)
 
                 # Delete downloaded file
                 rm(filepath)
@@ -517,7 +534,7 @@ function main!(content::OutputJsonContent, dest::OutputDestination)
                         if (ex isa HTTP.Exceptions.StatusError) && (ex.status == 404)
                             println(stdout, " ✗")
                             mark_url_as_nonexistent!(content, version, url)
-                            checkpoint(content, dest)
+                            checkpoint(content, cfg)
                             continue # skip the rest of this for-loop iteration
                         else
                             rethrow()
@@ -529,13 +546,17 @@ function main!(content::OutputJsonContent, dest::OutputDestination)
                 # Add in `.asc` signature content, if applicable
                 if !isnothing(asc_signature)
                     file_dict["asc"] = asc_signature
-                    checkpoint(content, dest)
+                    checkpoint(content, cfg)
                 end
             end
         end
     end
     @info "Tried $(number_urls_tried) versions, successfully downloaded $(number_urls_success). Skipped $num_urls_already_complete already good. Skipped $(number_urls_already_known_nonexistent) known bad."
-    return dest
+
+    return_value = (;
+        cfg.versions_json_filename,
+    )
+    return cfg
 end
 
 end # module