Stuff

Author: DilumAluthge

.github/workflows/CI.yml

@@ -49,7 +49,7 @@ jobs:
     # This job takes a long time (1+ hours).
     # So we intentionally skip this job on PR runs.
     # But we still run this job on Merge Queue jobs.
-    if: github.event_name != 'pull_request'
+    # if: github.event_name != 'pull_request' # TODO: Uncomment this line before merging the PR
     runs-on: ubuntu-latest
     timeout-minutes: 360
     strategy:
@@ -69,7 +69,7 @@ jobs:
           make clean
           make nuke
           make purge_download_cache_dir
-      - name: Download the old versions.json, so we can re-use it
+      - name: Download the old versions.json, so we can re-use it and build on top of it
         run: curl -LO https://julialang-s3.julialang.org/bin/versions.json
       - name: Build versions.json
         run: make build
@@ -95,7 +95,6 @@ jobs:
     if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
-
     steps:
       - name: Download versions.json from previous job
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -112,7 +111,7 @@ jobs:
       - name: Purge Fastly cache (Fastly sits in front of S3)
         run: curl -X PURGE https://julialang-s3.julialang.org/bin/versions.json
 
-  test-current-s3:
+  test-current-s3-allowed-to-fail:
     # We intentionally do not make this job a "required status check" on PRs.
     # Because if the currently deployed `versions.json` on S3 is already broken,
     # then this job will fail, which would block us from merging a PR with fixes.

src/VersionsJSONUtil.jl

@@ -548,12 +548,12 @@ function main!(content::OutputJsonContent, cfg::Config)
                 # Download this URL to a local file in a temp directory
                 number_urls_tried += 1
 
-                # This will download the file to a tempfile, which will automatically
-                # be cleaned up when the Julia process exits.
+                # This will download the file to a tempfile
                 filepath = Downloads.download(string(url))
 
                 if filesize(filepath) == 0
                     # The file is empty
+                    rm(filepath; force = true)
                     update_download_progress!(version, filename; advance = true)
                     continue # skip the rest of this for-loop iteration
                 end
@@ -622,6 +622,9 @@ function main!(content::OutputJsonContent, cfg::Config)
                 # Let's be forward-thinking: Make this an array of dictionaries that is
                 # easy to extensibly match.
                 push!(meta[version].files, FileDict(file_dict))
+
+                # Delete the downloaded file
+                rm(filepath; force = true)
             end
 
             file_dict = find_filedict(content, version, url)
@@ -640,11 +643,12 @@ function main!(content::OutputJsonContent, cfg::Config)
                     asc_filename = basename(string(asc_url))
                     @debug "Downloading $asc_filename"
                     try
-                        asc_filepath = download_to_cache(asc_filename, asc_url)
+                        asc_filepath = Downloads.download(string(asc_url))
                         asc_signature = String(read(asc_filepath))
+                        rm(asc_filepath)
                     catch ex
-                        if (ex isa HTTP.Exceptions.StatusError) && (ex.status == 404)
-                            # mark_url_as_nonexistent!(content, version, url) # TODO: Decide if we want to mark
+                        if (ex isa Downloads.RequestError) && (ex.response.status == 404)
+                            mark_url_as_nonexistent!(content, version, url)
                             update_download_progress!(version, filename; advance = true)
                             continue # skip the rest of this for-loop iteration
                         else