From 47a245ccddd413f9ba957ee53ff1ce22cf8f8338 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Fri, 19 Sep 2025 16:26:45 -0300 Subject: [PATCH 01/11] add workflow with criterion execution benches --- .github/workflows/bench-criterion.yml | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/bench-criterion.yml diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml new file mode 100644 index 0000000000..a5aca3c289 --- /dev/null +++ b/.github/workflows/bench-criterion.yml @@ -0,0 +1,74 @@ +name: Bench + +on: + pull_request: + branches: [main] + merge_group: + types: [checks_requested] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + bench-criterion: + name: Criterion + runs-on: ubuntu-24.04 + strategy: + matrix: + branchl: [base, head] + env: + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ + TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ + LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ + steps: + - uses: actions/checkout@v4 + # Install dependencies and free space + - uses: ./cairo_native/.github/actions/install-linux-deps + + - name: Setup rust env + uses: dtolnay/rust-toolchain@1.89.0 + + - name: Retreive cached dependecies + uses: Swatinem/rust-cache@v2 + + - name: Install deps + run: make deps + + - name: Install critcmp + run: cargo install critcmp + + - name: Checkout commit ${{ matrix.branch }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request[matrix.branch].sha }} + + - name: Run execution benches in ${{ matrix.branch }} + run: cargo bench --bench benches -- --save-baseline ${{ matrix.branch }} + + - name: Compre HEAD vs BASE + run: critcmp base head | tee bench-results.md + + - name: Find Bench Comment + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Criterion results Main vs HEAD + + - name: Create or update bench comment + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: bench-results.md + edit-mode: replace + + From 8fdc9032282acbdf6225756c6399f6019e30f11c Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Fri, 19 Sep 2025 16:34:33 -0300 Subject: [PATCH 02/11] add path to native --- .github/workflows/bench-criterion.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index a5aca3c289..fde409b863 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -26,7 +26,10 @@ jobs: TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ steps: - - uses: actions/checkout@v4 + - name: Checkout Native + uses: actions/checkout@v4 + with: + path: cairo_native # Install dependencies and free space - uses: ./cairo_native/.github/actions/install-linux-deps From 717c849adbde77b9207f2b93b036b5120f0853b4 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Fri, 19 Sep 2025 18:18:41 -0300 Subject: [PATCH 03/11] try fix --- .github/workflows/bench-criterion.yml | 46 ++++++++++++--------------- benches/benches.rs | 10 +++--- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index fde409b863..2679ed6768 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -14,48 +14,44 @@ env: CARGO_TERM_COLOR: always jobs: - bench-criterion: - name: Criterion + run-criterion-bench: + name: Run Criterion Bench runs-on: ubuntu-24.04 - strategy: - matrix: - branchl: [base, head] env: - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ steps: - - name: Checkout Native + - name: Checkout commit uses: actions/checkout@v4 with: path: cairo_native + # Install dependencies and free space - - uses: ./cairo_native/.github/actions/install-linux-deps - + - name: Install linux deps + uses: ./cairo_native/.github/actions/install-linux-deps + - name: Setup rust env uses: dtolnay/rust-toolchain@1.89.0 - - name: Retreive cached dependecies - uses: Swatinem/rust-cache@v2 - - - name: Install deps - run: make deps - - - name: Install critcmp - run: cargo install critcmp + - name: Run head bench + run: cargo bench --bench benches -- --save-baseline bench-head - - name: Checkout commit ${{ matrix.branch }} + - name: Checkout commit uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request[matrix.branch].sha }} - - - name: Run execution benches in ${{ matrix.branch }} - run: cargo bench --bench benches -- --save-baseline ${{ matrix.branch }} - + path: cairo_native + ref: main + + - name: Run base bench + run: cargo bench --bench benches -- --save-baseline bench-base + + - name: Install critcmp + run: cargo install critcmp + - name: Compre HEAD vs BASE - run: critcmp base head | tee bench-results.md - + run: critcmp bench-base bench-head | tee bench-results.md + - name: Find Bench Comment continue-on-error: true uses: peter-evans/find-comment@v3 diff --git a/benches/benches.rs b/benches/benches.rs index 066c58909d..98cef9be12 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -85,11 +85,11 @@ fn compare(c: &mut Criterion, path: impl AsRef) { fn criterion_benchmark(c: &mut Criterion) { compare(c, "programs/benches/dict_snapshot.cairo"); - compare(c, "programs/benches/dict_insert.cairo"); - compare(c, "programs/benches/factorial_2M.cairo"); - compare(c, "programs/benches/fib_2M.cairo"); - compare(c, "programs/benches/linear_search.cairo"); - compare(c, "programs/benches/logistic_map.cairo"); + // compare(c, "programs/benches/dict_insert.cairo"); + // compare(c, "programs/benches/factorial_2M.cairo"); + // compare(c, "programs/benches/fib_2M.cairo"); + // compare(c, "programs/benches/linear_search.cairo"); + // compare(c, "programs/benches/logistic_map.cairo"); } fn load_contract(path: impl AsRef) -> Program { From 1975c61f570ee5c2f3483a92f1adc863e88e5e96 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Mon, 22 Sep 2025 09:28:08 -0300 Subject: [PATCH 04/11] cd into cairo_native before running bench --- .github/workflows/bench-criterion.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index 2679ed6768..ca3eb1a614 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -35,7 +35,9 @@ jobs: uses: dtolnay/rust-toolchain@1.89.0 - name: Run head bench - run: cargo bench --bench benches -- --save-baseline bench-head + run: | + cd cairo_native + cargo bench --bench benches -- --save-baseline bench-head - name: Checkout commit uses: actions/checkout@v4 @@ -44,7 +46,9 @@ jobs: ref: main - name: Run base bench - run: cargo bench --bench benches -- --save-baseline bench-base + run: | + cd cairo_native + cargo bench --bench benches -- --save-baseline bench-base - name: Install critcmp run: cargo install critcmp From 7a25284f470fbd258bf2c23ba816fc2742bead8f Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Mon, 22 Sep 2025 10:12:58 -0300 Subject: [PATCH 05/11] install deps --- .github/workflows/bench-criterion.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index ca3eb1a614..8532943b3c 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -31,8 +31,11 @@ jobs: - name: Install linux deps uses: ./cairo_native/.github/actions/install-linux-deps - - name: Setup rust env - uses: dtolnay/rust-toolchain@1.89.0 + - uses: dtolnay/rust-toolchain@1.89.0 + - uses: Swatinem/rust-cache@v2 + + - name: install deps + run: make deps - name: Run head bench run: | From f94a8e7600be484366b651643c43789b32ff3acc Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Mon, 22 Sep 2025 10:35:56 -0300 Subject: [PATCH 06/11] add default workspace --- .github/workflows/bench-criterion.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index 8532943b3c..115c596704 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -21,6 +21,10 @@ jobs: MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ + defaults: + run: + shell: bash + working-directory: ./cairo_native steps: - name: Checkout commit uses: actions/checkout@v4 @@ -38,9 +42,7 @@ jobs: run: make deps - name: Run head bench - run: | - cd cairo_native - cargo bench --bench benches -- --save-baseline bench-head + run: cargo bench --bench benches -- --save-baseline bench-head - name: Checkout commit uses: actions/checkout@v4 @@ -49,9 +51,7 @@ jobs: ref: main - name: Run base bench - run: | - cd cairo_native - cargo bench --bench benches -- --save-baseline bench-base + run: cargo bench --bench benches -- --save-baseline bench-base - name: Install critcmp run: cargo install critcmp From e6869a2cff6b77365d7eda12ad0a9afb9c185d49 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Mon, 22 Sep 2025 11:07:37 -0300 Subject: [PATCH 07/11] uncomment --- benches/benches.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benches/benches.rs b/benches/benches.rs index 98cef9be12..066c58909d 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -85,11 +85,11 @@ fn compare(c: &mut Criterion, path: impl AsRef) { fn criterion_benchmark(c: &mut Criterion) { compare(c, "programs/benches/dict_snapshot.cairo"); - // compare(c, "programs/benches/dict_insert.cairo"); - // compare(c, "programs/benches/factorial_2M.cairo"); - // compare(c, "programs/benches/fib_2M.cairo"); - // compare(c, "programs/benches/linear_search.cairo"); - // compare(c, "programs/benches/logistic_map.cairo"); + compare(c, "programs/benches/dict_insert.cairo"); + compare(c, "programs/benches/factorial_2M.cairo"); + compare(c, "programs/benches/fib_2M.cairo"); + compare(c, "programs/benches/linear_search.cairo"); + compare(c, "programs/benches/logistic_map.cairo"); } fn load_contract(path: impl AsRef) -> Program { From a255a4426579d7eae9c21c3a0332e1c1a8554997 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Mon, 22 Sep 2025 11:22:45 -0300 Subject: [PATCH 08/11] uncomment benches and install deps for base --- .github/workflows/bench-criterion.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index 115c596704..3630ca8c29 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -38,7 +38,7 @@ jobs: - uses: dtolnay/rust-toolchain@1.89.0 - uses: Swatinem/rust-cache@v2 - - name: install deps + - name: install deps head run: make deps - name: Run head bench @@ -50,6 +50,9 @@ jobs: path: cairo_native ref: main + - name: install deps base + run: make deps + - name: Run base bench run: cargo bench --bench benches -- --save-baseline bench-base From 7ce08832609762e80bd176271680691e346c5173 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Wed, 24 Sep 2025 15:44:23 -0300 Subject: [PATCH 09/11] output to a file --- .github/workflows/bench-criterion.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index 3630ca8c29..e4e38a3ed7 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -60,7 +60,7 @@ jobs: run: cargo install critcmp - name: Compre HEAD vs BASE - run: critcmp bench-base bench-head | tee bench-results.md + run: critcmp bench-base bench-head > bench-results.md - name: Find Bench Comment continue-on-error: true From e5e0fdf21e5bc721c46597c73792eb27fdcaa237 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Wed, 24 Sep 2025 16:30:01 -0300 Subject: [PATCH 10/11] add testing flag to bench --- .github/workflows/bench-criterion.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index e4e38a3ed7..87e951fed5 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -42,7 +42,7 @@ jobs: run: make deps - name: Run head bench - run: cargo bench --bench benches -- --save-baseline bench-head + run: cargo bench -F testing --bench benches -- --save-baseline bench-head - name: Checkout commit uses: actions/checkout@v4 @@ -54,7 +54,7 @@ jobs: run: make deps - name: Run base bench - run: cargo bench --bench benches -- --save-baseline bench-base + run: cargo bench -F testing --bench benches -- --save-baseline bench-base - name: Install critcmp run: cargo install critcmp From 187adf4bfd20b40e6e2622ac7219fc4fe0857ef2 Mon Sep 17 00:00:00 2001 From: FrancoGiachetta Date: Fri, 26 Sep 2025 15:31:39 -0300 Subject: [PATCH 11/11] changes --- .github/workflows/bench-criterion.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bench-criterion.yml b/.github/workflows/bench-criterion.yml index 87e951fed5..f9853452be 100644 --- a/.github/workflows/bench-criterion.yml +++ b/.github/workflows/bench-criterion.yml @@ -60,8 +60,17 @@ jobs: run: cargo install critcmp - name: Compre HEAD vs BASE - run: critcmp bench-base bench-head > bench-results.md - + run: critcmp bench-base bench-head | tee -a comment_body.md + + - name: Create or update bench comment + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.pull_request.number }} + issue-number: ${{ github.event.pull_request.number }} + body-path: comment_body.md + edit-mode: replace + - name: Find Bench Comment continue-on-error: true uses: peter-evans/find-comment@v3 @@ -70,14 +79,5 @@ jobs: issue-number: ${{ github.event.pull_request.number }} comment-author: "github-actions[bot]" body-includes: Criterion results Main vs HEAD - - - name: Create or update bench comment - continue-on-error: true - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: bench-results.md - edit-mode: replace