From cfe36584a2ac2e482c3a6adffdb89df757144ce0 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 9 May 2026 18:09:53 -0400 Subject: [PATCH 1/5] Debug workflow failure --- .github/workflows/process_results.yaml | 13 +++++++++---- .github/workflows/run_asvs_2026_01_04.yaml | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml index e6df943d147..b55e321718d 100644 --- a/.github/workflows/process_results.yaml +++ b/.github/workflows/process_results.yaml @@ -40,12 +40,12 @@ jobs: ref: "${{ env.BRANCH_NAME }}" path: asv-runner/ - # ci/ scripts are run from main, not from whatever stale copy lives - # on the storage branch. - - name: Checkout asv-runner main branch + # ci/ scripts are run from the workflow's own ref (main on schedule, + # whatever branch you choose on workflow_dispatch), not from whatever + # stale copy lives on the storage branch. + - name: Checkout asv-runner code uses: actions/checkout@v6 with: - ref: main path: asv-runner-code/ # Migrate from legacy loose uncompressed JSON/YAML files to per-file @@ -111,6 +111,11 @@ jobs: # tree, then orphan force-push. Retry on lease failure. - name: Push results run: | + # Trace every command with its line number so we can pinpoint the + # failing line on the next run. Remove once the failure is fixed. + PS4='+ [line:$LINENO] ' + set -x + SAVE=$(mktemp -d) cp -r asv-runner/data/results.parquet "$SAVE/" cp -r asv-runner/docs "$SAVE/" diff --git a/.github/workflows/run_asvs_2026_01_04.yaml b/.github/workflows/run_asvs_2026_01_04.yaml index 2d71745626a..79b37da21e7 100644 --- a/.github/workflows/run_asvs_2026_01_04.yaml +++ b/.github/workflows/run_asvs_2026_01_04.yaml @@ -47,12 +47,12 @@ jobs: ref: ${{ env.BRANCH_NAME }} path: asv-runner/ - # ci/ scripts are run from main, not from whatever stale copy lives - # on the storage branch. The storage branch is data-only. - - name: Checkout asv-runner main branch + # ci/ scripts are run from the workflow's own ref (main on schedule, + # whatever branch you choose on workflow_dispatch), not from whatever + # stale copy lives on the storage branch. The storage branch is data-only. + - name: Checkout asv-runner code uses: actions/checkout@v6 with: - ref: main path: asv-runner-code/ - name: Claim next SHA From 8393cd4bfdf803896aee825b03a4f751ebf51efc Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 9 May 2026 18:55:13 -0400 Subject: [PATCH 2/5] More debug --- .github/workflows/process_results.yaml | 4 ++-- ci/process_results.py | 32 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml index b55e321718d..6bf178205ef 100644 --- a/.github/workflows/process_results.yaml +++ b/.github/workflows/process_results.yaml @@ -91,8 +91,8 @@ jobs: find asv-runner/data/envs -name '*.yml.zst' -print0 | xargs -0 -r zstd -d -q --output-dir-flat asv-runner/data/envs - - name: Publish ASV Benchmarks - run: cd asv_bench && asv publish +# - name: Publish ASV Benchmarks +# run: cd asv_bench && asv publish - name: Process ASV results run: | diff --git a/ci/process_results.py b/ci/process_results.py index 13a50aaa538..10f885f36b5 100644 --- a/ci/process_results.py +++ b/ci/process_results.py @@ -2,13 +2,24 @@ import argparse import datetime as dt +import faulthandler import itertools as it import json +import sys from pathlib import Path import pandas as pd import pyarrow as pa +# Print a Python stack trace on SIGABRT / SIGSEGV / etc. so a crash inside +# pyarrow / pandas C extensions still surfaces a useful traceback. +faulthandler.enable() + + +def _step(msg: str) -> None: + print(f"[process_results] {msg}", flush=True) + + PARQUET_DIRNAME = "results.parquet" BASE_COLUMNS = ["date", "sha", "name", "params", "result", "added_date"] DERIVED_COLUMNS = [ @@ -21,25 +32,30 @@ def detect_regression(data: pd.DataFrame, window_size: int = 21) -> pd.DataFrame: + _step(f"detect_regression: input rows={len(data)}") data = ( data[data["result"].notnull()] .set_index(["name", "params", "date"]) .sort_index() ) + _step(f"detect_regression: after notnull+sort rows={len(data)}") keys = ["name", "params"] tol = 0.95 + _step("detect_regression: rolling max (established_worst)") data["established_worst"] = ( data.groupby(keys, as_index=False)["result"] .rolling(window_size, center=True) .max()[["result"]] ) + _step("detect_regression: rolling min (established_best)") data["established_best"] = ( data.groupby(keys, as_index=False)["result"] .rolling(window_size, center=True) .min()[["result"]] ) + _step("detect_regression: building mask") mask = ( # TODO: is the arg to shift right? data["established_worst"].groupby(keys).shift(window_size) @@ -49,8 +65,10 @@ def detect_regression(data: pd.DataFrame, window_size: int = 21) -> pd.DataFrame mask = mask.groupby(keys).shift(-(window_size - 1) // 2, fill_value=False) data["is_regression"] = mask + _step("detect_regression: pct_change/abs_change") data["pct_change"] = data.groupby(keys)["result"].pct_change() data["abs_change"] = data["result"] - data.groupby(keys)["result"].shift(1) + _step("detect_regression: done, resetting index") return data.reset_index() @@ -119,19 +137,28 @@ def build_new_rows( def run(input_path: str | Path, output_path: str | Path): + _step( + f"run: pandas={pd.__version__} pyarrow={pa.__version__} python={sys.version.split()[0]}" + ) input_path = Path(input_path) output_path = Path(output_path) parquet_path = output_path / PARQUET_DIRNAME + _step(f"run: load_existing({parquet_path})") existing = load_existing(parquet_path) + _step(f"run: existing rows={0 if existing is None else len(existing)}") skip_shas: set[str] = ( set(existing["sha"].dropna().unique()) if existing is not None else set() ) + _step(f"run: skip_shas count={len(skip_shas)}") today = dt.datetime.now(dt.timezone.utc).date().isoformat() + _step(f"run: build_new_rows (added_date={today})") new_rows = build_new_rows(input_path, skip_shas, today) + _step(f"run: new rows={len(new_rows)}") if existing is not None: + _step("run: concat existing + new") existing_base = existing.drop( columns=[c for c in DERIVED_COLUMNS if c in existing.columns] ) @@ -140,9 +167,13 @@ def run(input_path: str | Path, output_path: str | Path): ) else: df = new_rows[BASE_COLUMNS] + _step(f"run: total rows={len(df)}") + _step("run: detect_regression") result = detect_regression(df, window_size=21) + _step(f"run: detected, regressions={int(result['is_regression'].sum())}") + _step(f"run: to_parquet({parquet_path})") result.to_parquet( parquet_path, index=False, @@ -150,6 +181,7 @@ def run(input_path: str | Path, output_path: str | Path): existing_data_behavior="delete_matching", basename_template="part-{i}.parquet", ) + _step("run: done") if __name__ == "__main__": From d46a6873e1a1d5f3281a67db7cf1b2c25a9c9c4c Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 9 May 2026 18:58:08 -0400 Subject: [PATCH 3/5] Revert --- .github/workflows/process_results.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml index 6bf178205ef..b55e321718d 100644 --- a/.github/workflows/process_results.yaml +++ b/.github/workflows/process_results.yaml @@ -91,8 +91,8 @@ jobs: find asv-runner/data/envs -name '*.yml.zst' -print0 | xargs -0 -r zstd -d -q --output-dir-flat asv-runner/data/envs -# - name: Publish ASV Benchmarks -# run: cd asv_bench && asv publish + - name: Publish ASV Benchmarks + run: cd asv_bench && asv publish - name: Process ASV results run: | From e9bff1bb0a5af935758f0ed8655812af4cf376f1 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 9 May 2026 19:20:50 -0400 Subject: [PATCH 4/5] fixup --- .github/workflows/process_results.yaml | 8 ++++---- .github/workflows/run_asvs_2026_01_04.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml index b55e321718d..d5fb86e13fb 100644 --- a/.github/workflows/process_results.yaml +++ b/.github/workflows/process_results.yaml @@ -54,11 +54,11 @@ jobs: run: | if [ -d asv-runner/data/results/asvrunner ]; then find asv-runner/data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi if [ -d asv-runner/data/envs ]; then find asv-runner/data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi - name: Setup Python @@ -131,11 +131,11 @@ jobs: # Idempotent. if [ -d data/results/asvrunner ]; then find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi if [ -d data/envs ]; then find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi # Restore our parquet+docs over the freshly fetched tree. diff --git a/.github/workflows/run_asvs_2026_01_04.yaml b/.github/workflows/run_asvs_2026_01_04.yaml index 79b37da21e7..44a08719f51 100644 --- a/.github/workflows/run_asvs_2026_01_04.yaml +++ b/.github/workflows/run_asvs_2026_01_04.yaml @@ -74,11 +74,11 @@ jobs: # produces no input. if [ -d data/results/asvrunner ]; then find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi if [ -d data/envs ]; then find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi cd .. @@ -217,11 +217,11 @@ jobs: # Idempotent. if [ -d data/results/asvrunner ]; then find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi if [ -d data/envs ]; then find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q + xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f fi # Drop in this run's new files. From 5ea2044a07153be7ffa0f7bfa3ca774fec3e7349 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 9 May 2026 19:41:54 -0400 Subject: [PATCH 5/5] Finish up --- .github/workflows/process_results.yaml | 31 +-------------------- .github/workflows/run_asvs_2026_01_04.yaml | 30 ++------------------ ci/process_results.py | 32 ---------------------- 3 files changed, 4 insertions(+), 89 deletions(-) diff --git a/.github/workflows/process_results.yaml b/.github/workflows/process_results.yaml index d5fb86e13fb..c958236bb97 100644 --- a/.github/workflows/process_results.yaml +++ b/.github/workflows/process_results.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-24.04 defaults: run: - shell: bash -el {0} + shell: bash -e {0} permissions: contents: write issues: write @@ -48,19 +48,6 @@ jobs: with: path: asv-runner-code/ - # Migrate from legacy loose uncompressed JSON/YAML files to per-file - # zstd. Idempotent. - - name: Migrate legacy storage layout - run: | - if [ -d asv-runner/data/results/asvrunner ]; then - find asv-runner/data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - if [ -d asv-runner/data/envs ]; then - find asv-runner/data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - - name: Setup Python uses: actions/setup-python@v6 with: @@ -111,11 +98,6 @@ jobs: # tree, then orphan force-push. Retry on lease failure. - name: Push results run: | - # Trace every command with its line number so we can pinpoint the - # failing line on the next run. Remove once the failure is fixed. - PS4='+ [line:$LINENO] ' - set -x - SAVE=$(mktemp -d) cp -r asv-runner/data/results.parquet "$SAVE/" cp -r asv-runner/docs "$SAVE/" @@ -127,17 +109,6 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # Migrate from legacy loose uncompressed files to per-file zstd. - # Idempotent. - if [ -d data/results/asvrunner ]; then - find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - if [ -d data/envs ]; then - find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - # Restore our parquet+docs over the freshly fetched tree. rm -rf data/results.parquet docs cp -r "$SAVE/results.parquet" data/results.parquet diff --git a/.github/workflows/run_asvs_2026_01_04.yaml b/.github/workflows/run_asvs_2026_01_04.yaml index 44a08719f51..4a170fed79c 100644 --- a/.github/workflows/run_asvs_2026_01_04.yaml +++ b/.github/workflows/run_asvs_2026_01_04.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-24.04 defaults: run: - shell: bash -el {0} + shell: bash -e {0} permissions: contents: write outputs: @@ -68,19 +68,6 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # Migrate from legacy loose uncompressed JSON/YAML files to - # per-file zstd. Idempotent: subsequent runs find no *.json - # (besides machine.json, excluded) or *.yml and the find - # produces no input. - if [ -d data/results/asvrunner ]; then - find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - if [ -d data/envs ]; then - find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - cd .. sha="$(python3 asv-runner-code/ci/find_commit_to_run.py --input-path=asv-runner/data/ --repo-path=.)" if [ "$sha" = "NONE" ]; then @@ -129,7 +116,7 @@ jobs: runs-on: ubuntu-24.04 defaults: run: - shell: bash -el {0} + shell: bash -e {0} env: SHA: ${{ needs.claim.outputs.sha }} steps: @@ -186,7 +173,7 @@ jobs: runs-on: ubuntu-24.04 defaults: run: - shell: bash -el {0} + shell: bash -e {0} permissions: contents: write env: @@ -213,17 +200,6 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # Migrate from legacy loose uncompressed files to per-file zstd. - # Idempotent. - if [ -d data/results/asvrunner ]; then - find data/results/asvrunner -name '*.json' ! -name 'machine.json' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - if [ -d data/envs ]; then - find data/envs -name '*.yml' -print0 | - xargs -0 -r -P 4 -n 50 zstd --rm -19 -q -f - fi - # Drop in this run's new files. mkdir -p data/results/asvrunner data/envs cp /tmp/new/results/benchmarks.json data/results/ diff --git a/ci/process_results.py b/ci/process_results.py index 10f885f36b5..13a50aaa538 100644 --- a/ci/process_results.py +++ b/ci/process_results.py @@ -2,24 +2,13 @@ import argparse import datetime as dt -import faulthandler import itertools as it import json -import sys from pathlib import Path import pandas as pd import pyarrow as pa -# Print a Python stack trace on SIGABRT / SIGSEGV / etc. so a crash inside -# pyarrow / pandas C extensions still surfaces a useful traceback. -faulthandler.enable() - - -def _step(msg: str) -> None: - print(f"[process_results] {msg}", flush=True) - - PARQUET_DIRNAME = "results.parquet" BASE_COLUMNS = ["date", "sha", "name", "params", "result", "added_date"] DERIVED_COLUMNS = [ @@ -32,30 +21,25 @@ def _step(msg: str) -> None: def detect_regression(data: pd.DataFrame, window_size: int = 21) -> pd.DataFrame: - _step(f"detect_regression: input rows={len(data)}") data = ( data[data["result"].notnull()] .set_index(["name", "params", "date"]) .sort_index() ) - _step(f"detect_regression: after notnull+sort rows={len(data)}") keys = ["name", "params"] tol = 0.95 - _step("detect_regression: rolling max (established_worst)") data["established_worst"] = ( data.groupby(keys, as_index=False)["result"] .rolling(window_size, center=True) .max()[["result"]] ) - _step("detect_regression: rolling min (established_best)") data["established_best"] = ( data.groupby(keys, as_index=False)["result"] .rolling(window_size, center=True) .min()[["result"]] ) - _step("detect_regression: building mask") mask = ( # TODO: is the arg to shift right? data["established_worst"].groupby(keys).shift(window_size) @@ -65,10 +49,8 @@ def detect_regression(data: pd.DataFrame, window_size: int = 21) -> pd.DataFrame mask = mask.groupby(keys).shift(-(window_size - 1) // 2, fill_value=False) data["is_regression"] = mask - _step("detect_regression: pct_change/abs_change") data["pct_change"] = data.groupby(keys)["result"].pct_change() data["abs_change"] = data["result"] - data.groupby(keys)["result"].shift(1) - _step("detect_regression: done, resetting index") return data.reset_index() @@ -137,28 +119,19 @@ def build_new_rows( def run(input_path: str | Path, output_path: str | Path): - _step( - f"run: pandas={pd.__version__} pyarrow={pa.__version__} python={sys.version.split()[0]}" - ) input_path = Path(input_path) output_path = Path(output_path) parquet_path = output_path / PARQUET_DIRNAME - _step(f"run: load_existing({parquet_path})") existing = load_existing(parquet_path) - _step(f"run: existing rows={0 if existing is None else len(existing)}") skip_shas: set[str] = ( set(existing["sha"].dropna().unique()) if existing is not None else set() ) - _step(f"run: skip_shas count={len(skip_shas)}") today = dt.datetime.now(dt.timezone.utc).date().isoformat() - _step(f"run: build_new_rows (added_date={today})") new_rows = build_new_rows(input_path, skip_shas, today) - _step(f"run: new rows={len(new_rows)}") if existing is not None: - _step("run: concat existing + new") existing_base = existing.drop( columns=[c for c in DERIVED_COLUMNS if c in existing.columns] ) @@ -167,13 +140,9 @@ def run(input_path: str | Path, output_path: str | Path): ) else: df = new_rows[BASE_COLUMNS] - _step(f"run: total rows={len(df)}") - _step("run: detect_regression") result = detect_regression(df, window_size=21) - _step(f"run: detected, regressions={int(result['is_regression'].sum())}") - _step(f"run: to_parquet({parquet_path})") result.to_parquet( parquet_path, index=False, @@ -181,7 +150,6 @@ def run(input_path: str | Path, output_path: str | Path): existing_data_behavior="delete_matching", basename_template="part-{i}.parquet", ) - _step("run: done") if __name__ == "__main__":