Use per-file compression (#131)

rhshadrach · web-flow · commit df3c3395032d · 2026-05-09T17:14:12.000-04:00
diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml
@@ -48,17 +48,18 @@ jobs:
           ref: main
           path: asv-runner-code/
 
-      # Idempotent migration from the legacy loose-files layout to compressed
-      # tarballs. No-op once the tarballs exist.
+      # Migrate from legacy loose uncompressed JSON/YAML files to per-file
+      # zstd. Idempotent.
       - name: Migrate legacy storage layout
         run: |
-          if [ -d asv-runner/data/results ] && [ ! -f asv-runner/data/results.tar.zst ]; then
-            tar -C asv-runner/data -I 'zstd -19' -cf asv-runner/data/results.tar.zst results
+          if [ -d asv-runner/data/results/asvrunner ]; then
+            find asv-runner/data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 |
+              xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
           fi
-          if [ -d asv-runner/data/envs ] && [ ! -f asv-runner/data/envs.tar.zst ]; then
-            tar -C asv-runner/data -I 'zstd -19' -cf asv-runner/data/envs.tar.zst envs
+          if [ -d asv-runner/data/envs ]; then
+            find asv-runner/data/envs -name '*.yml' -print0 |
+              xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
           fi
-          rm -rf asv-runner/data/results asv-runner/data/envs
 
       - name: Setup Python
         uses: actions/setup-python@v6
@@ -74,14 +75,21 @@ jobs:
       - name: Show environment packages
         run: uv pip freeze
 
-      # Extract results.tar.zst into asv_bench/ so `asv publish` and
-      # process_results.py see the same paths as before. Extract envs.tar.zst
-      # back under asv-runner/data/envs/ so make_issues.py can read per-SHA
-      # env files. The extracted dirs are gitignored.
-      - name: Extract result and env archives
+      # Decompress per-SHA results into asv_bench/results/asvrunner/ where
+      # asv publish and process_results.py expect them. benchmarks.json
+      # and machine.json are stored uncompressed and copied as-is. Envs are
+      # decompressed in place under data/envs/ so make_issues.py can read
+      # <sha>.yml directly; the decompressed *.yml files are gitignored.
+      # Single batched zstd invocation per group amortizes process startup.
+      - name: Decompress results and envs
         run: |
-          tar -I zstd -xf asv-runner/data/results.tar.zst -C asv_bench/
-          tar -I zstd -xf asv-runner/data/envs.tar.zst -C asv-runner/data/
+          mkdir -p asv_bench/results/asvrunner
+          cp asv-runner/data/results/benchmarks.json asv_bench/results/
+          cp asv-runner/data/results/asvrunner/machine.json asv_bench/results/asvrunner/
+          find asv-runner/data/results/asvrunner -name '*.json.zst' -print0 |
+            xargs -0 -r zstd -d -q --output-dir-flat asv_bench/results/asvrunner
+          find asv-runner/data/envs -name '*.yml.zst' -print0 |
+            xargs -0 -r zstd -d -q --output-dir-flat asv-runner/data/envs
 
       - name: Publish ASV Benchmarks
         run: cd asv_bench && asv publish
@@ -97,10 +105,10 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      # Save our parquet+docs, refetch the latest tarballs (a concurrent
-      # run_asvs merge may have updated them while asv publish was running),
-      # restore our parquet+docs over the fresh tarballs, then orphan
-      # force-push. Retry on lease failure.
+      # Save our parquet+docs, refetch the latest tree (a concurrent
+      # run_asvs merge may have added new <sha>.json.zst files while
+      # asv publish was running), restore our parquet+docs over the fresh
+      # tree, then orphan force-push. Retry on lease failure.
       - name: Push results
         run: |
           SAVE=$(mktemp -d)
@@ -114,16 +122,18 @@ jobs:
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
 
-            # Idempotent migration from legacy loose-files layout.
-            if [ -d data/results ] && [ ! -f data/results.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/results.tar.zst results
+            # Migrate from legacy loose uncompressed files to per-file zstd.
+            # Idempotent.
+            if [ -d data/results/asvrunner ]; then
+              find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            if [ -d data/envs ] && [ ! -f data/envs.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/envs.tar.zst envs
+            if [ -d data/envs ]; then
+              find data/envs -name '*.yml' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            rm -rf data/results data/envs
 
-            # Restore our parquet+docs over the freshly fetched tarballs.
+            # Restore our parquet+docs over the freshly fetched tree.
             rm -rf data/results.parquet docs
             cp -r "$SAVE/results.parquet" data/results.parquet
             cp -r "$SAVE/docs" docs
diff --git a/.github/workflows/run_asvs_2026_01_04.yaml b/.github/workflows/run_asvs_2026_01_04.yaml
@@ -68,14 +68,18 @@ jobs:
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
 
-            # One-shot migration from the legacy loose-files layout. Idempotent.
-            if [ -d data/results ] && [ ! -f data/results.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/results.tar.zst results
+            # Migrate from legacy loose uncompressed JSON/YAML files to
+            # per-file zstd. Idempotent: subsequent runs find no *.json
+            # (besides machine.json, excluded) or *.yml and the find
+            # produces no input.
+            if [ -d data/results/asvrunner ]; then
+              find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            if [ -d data/envs ] && [ ! -f data/envs.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/envs.tar.zst envs
+            if [ -d data/envs ]; then
+              find data/envs -name '*.yml' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            rm -rf data/results data/envs
 
             cd ..
             sha="$(python3 asv-runner-code/ci/find_commit_to_run.py --input-path=asv-runner/data/ --repo-path=.)"
@@ -116,7 +120,8 @@ jobs:
 
   # Build pandas at the claimed SHA and run the ASV suite. No concurrency
   # lock here — multiple workflow runs benchmark different SHAs in parallel.
-  # Outputs are uploaded as a workflow artifact for the merge job.
+  # The new <sha>.json and <sha>.yml are zstd-compressed before upload so
+  # the merge job can drop them into the storage tree as-is.
   benchmark:
     name: Run benchmarks
     needs: claim
@@ -160,17 +165,20 @@ jobs:
           cp asv_bench/results/asvrunner/machine.json /tmp/upload/results/asvrunner/
           cp asv_bench/results/asvrunner/${SHORT_SHA}-existing*.json /tmp/upload/results/asvrunner/${SHA}.json
           pixi list -e asv --fields name,version > /tmp/upload/envs/${SHA}.yml
+          # Compress per-SHA files. benchmarks.json and machine.json stay
+          # uncompressed (small, overwritten each run, read directly).
+          zstd --rm -19 -q /tmp/upload/results/asvrunner/${SHA}.json
+          zstd --rm -19 -q /tmp/upload/envs/${SHA}.yml
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: asv-results-${{ needs.claim.outputs.sha }}
           path: /tmp/upload/
 
-  # Layer this run's results into the latest tarballs and orphan force-push.
-  # If a concurrent merge wins the race (lease fails), refetch — that
-  # winner's data is now in the tarballs we extract — re-add ours on top
-  # and try again.
+  # Drop this run's compressed files into the storage tree and orphan
+  # force-push. If a concurrent merge wins the race (lease fails), refetch
+  # — that winner's data is already on disk after the fetch — and redo.
   merge:
     name: Merge into storage branch
     needs: [claim, benchmark]
@@ -205,31 +213,23 @@ jobs:
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
 
-            # Idempotent migration from legacy loose-files layout.
-            if [ -d data/results ] && [ ! -f data/results.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/results.tar.zst results
+            # Migrate from legacy loose uncompressed files to per-file zstd.
+            # Idempotent.
+            if [ -d data/results/asvrunner ]; then
+              find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            if [ -d data/envs ] && [ ! -f data/envs.tar.zst ]; then
-              tar -C data -I 'zstd -19' -cf data/envs.tar.zst envs
+            if [ -d data/envs ]; then
+              find data/envs -name '*.yml' -print0 |
+                xargs -0 -r -P 4 -n 50 zstd --rm -19 -q
             fi
-            rm -rf data/results data/envs
 
-            WORK=$(mktemp -d)
-            mkdir -p "$WORK/results/asvrunner" "$WORK/envs"
-            if [ -f data/results.tar.zst ]; then
-              tar -I zstd -xf data/results.tar.zst -C "$WORK"
-            fi
-            if [ -f data/envs.tar.zst ]; then
-              tar -I zstd -xf data/envs.tar.zst -C "$WORK"
-            fi
-
-            cp /tmp/new/results/benchmarks.json "$WORK/results/"
-            cp /tmp/new/results/asvrunner/machine.json "$WORK/results/asvrunner/"
-            cp /tmp/new/results/asvrunner/${SHA}.json "$WORK/results/asvrunner/"
-            cp /tmp/new/envs/${SHA}.yml "$WORK/envs/"
-
-            tar -C "$WORK" -I 'zstd -19' -cf data/results.tar.zst results
-            tar -C "$WORK" -I 'zstd -19' -cf data/envs.tar.zst envs
+            # Drop in this run's new files.
+            mkdir -p data/results/asvrunner data/envs
+            cp /tmp/new/results/benchmarks.json data/results/
+            cp /tmp/new/results/asvrunner/machine.json data/results/asvrunner/
+            cp /tmp/new/results/asvrunner/${SHA}.json.zst data/results/asvrunner/
+            cp /tmp/new/envs/${SHA}.yml.zst data/envs/
 
             EXPECTED=$(git rev-parse origin/${BRANCH_NAME})
             git checkout --orphan fresh
diff --git a/data/.gitignore b/data/.gitignore
@@ -1,2 +1,5 @@
-results/
-envs/
+# Transient decompressed files: storage uses *.json.zst / *.yml.zst,
+# decompression at workflow time produces *.json / *.yml in place.
+results/asvrunner/*.json
+!results/asvrunner/machine.json
+envs/*.yml
