From c4a8e042acc751bed3cb4df1b30580fb0469de8b Mon Sep 17 00:00:00 2001 From: "Naor.d" Date: Sun, 29 Mar 2026 13:59:58 +0300 Subject: [PATCH 1/4] feat(makefile,ci): add cairo_test_suite_programs target and CI integration Makefile: - Add CAIRO_TEST_SUITE_ROOT, CAIRO_TEST_SUITE_FILES, COMPILED_CAIRO_TEST_SUITE variables to track cairo test suite sources - Add pattern rule to compile each .cairo to .json via cairo-compile - Add cairo_test_suite_programs target - Hook cairo_test_suite_programs into the test target - Add cairo_test_suite_programs to .PHONY CI (rust.yml): - Add cairo_test_suite_programs to the build-programs matrix - Add vm/src/tests/cairo_test_suite/**/*.json to CAIRO_PROGRAMS_PATH cache - Update all program cache keys to hash test suite source files - Add function_runner feature flag to the tests job Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/rust.yml | 40 +++++++++++++++++++++++--------------- Makefile | 13 +++++++++++-- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ade377346e..52210d6d11 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -18,6 +18,7 @@ env: cairo_programs/**/*.json !cairo_programs/manually_compiled/* cairo_programs/cairo-1-programs/bitwise.sierra + vm/src/tests/cairo_test_suite/**/*.json TEST_COLLECT_COVERAGE: 1 PROPTEST_CASES: 100 @@ -46,6 +47,7 @@ jobs: - cairo_bench_programs - cairo_proof_programs - cairo_test_programs + - cairo_test_suite_programs - cairo_1_test_contracts - cairo_2_test_contracts name: Build Cairo programs @@ -66,7 +68,7 @@ jobs: id: cache-programs with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} # This is not pretty, but we need `make` to see the compiled programs are # actually newer than the sources, otherwise it will try to rebuild them @@ -120,37 +122,43 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_test_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_test_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Fetch proof programs uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Fetch bench programs uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_bench_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_bench_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} + fail-on-cache-miss: true + - name: Fetch cairo test suite programs + uses: actions/cache/restore@v3 + with: + path: ${{ env.CAIRO_PROGRAMS_PATH }} + key: cairo_test_suite_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Fetch test contracts (Cairo 1) uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_1_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_1_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Fetch test contracts (Cairo 2) uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_2_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_2_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Merge caches uses: actions/cache/save@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} lint: needs: merge-caches @@ -179,7 +187,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Run clippy @@ -229,7 +237,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true # NOTE: we do this separately because --workspace operates in weird ways @@ -274,7 +282,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Check all features (workspace) @@ -309,7 +317,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Install testing tools @@ -329,7 +337,7 @@ jobs: export PARTITION=${MATRIX_TARGET#*#} cargo llvm-cov nextest --lcov --output-path lcov-${{ matrix.target }}-${{ matrix.special_features }}.info \ --partition count:${PARTITION}/4 \ - --workspace --features "cairo-1-hints, test_utils, ${{ matrix.special_features }}" + --workspace --features "cairo-1-hints, test_utils, function_runner, ${{ matrix.special_features }}" - name: Save coverage uses: actions/cache/save@v3 @@ -409,7 +417,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true # This is not pretty, but we need `make` to see the compiled programs are @@ -456,7 +464,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Generate traces @@ -675,7 +683,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Run script @@ -715,7 +723,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CAIRO_PROGRAMS_PATH }} - key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }} + key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }} fail-on-cache-miss: true - name: Fetch pie diff --git a/Makefile b/Makefile index 1734325614..dbf76a3ab2 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ UNAME := $(shell uname) compare_trace_memory compare_trace compare_memory compare_pie compare_all_no_proof \ compare_trace_memory_proof compare_all_proof compare_trace_proof compare_memory_proof compare_air_public_input compare_air_private_input\ hyper-threading-benchmarks \ - cairo_bench_programs cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_2_test_contracts \ + cairo_bench_programs cairo_proof_programs cairo_test_programs cairo_test_suite_programs cairo_1_test_contracts cairo_2_test_contracts \ cairo_trace cairo-vm_trace cairo_proof_trace cairo-vm_proof_trace python-deps python-deps-macos \ build-cairo-lang hint-accountant \ create-proof-programs-symlinks \ $(RELBIN) $(DBGBIN) @@ -124,6 +124,10 @@ NORETROCOMPAT_DIR:=cairo_programs/noretrocompat NORETROCOMPAT_FILES:=$(wildcard $(NORETROCOMPAT_DIR)/*.cairo) COMPILED_NORETROCOMPAT_TESTS:=$(patsubst $(NORETROCOMPAT_DIR)/%.cairo, $(NORETROCOMPAT_DIR)/%.json, $(NORETROCOMPAT_FILES)) +CAIRO_TEST_SUITE_ROOT:=vm/src/tests/cairo_test_suite +CAIRO_TEST_SUITE_FILES:=$(shell find $(CAIRO_TEST_SUITE_ROOT) -name "*.cairo") +COMPILED_CAIRO_TEST_SUITE:=$(patsubst %.cairo,%.json,$(CAIRO_TEST_SUITE_FILES)) + $(BENCH_DIR)/%.json: $(BENCH_DIR)/%.cairo cairo-compile --cairo_path="$(TEST_DIR):$(BENCH_DIR)" $< --output $@ --proof_mode @@ -145,6 +149,9 @@ $(BAD_TEST_DIR)/%.json: $(BAD_TEST_DIR)/%.cairo $(PRINT_TEST_DIR)/%.json: $(PRINT_TEST_DIR)/%.cairo cairo-compile $< --output $@ +$(CAIRO_TEST_SUITE_ROOT)/%.json: $(CAIRO_TEST_SUITE_ROOT)/%.cairo + cairo-compile $< --output $@ + # ====================== # Test Cairo 1 Contracts # ====================== @@ -268,6 +275,8 @@ run: check: cargo check +cairo_test_suite_programs: $(COMPILED_CAIRO_TEST_SUITE) + cairo_test_programs: $(COMPILED_TESTS) $(COMPILED_BAD_TESTS) $(COMPILED_NORETROCOMPAT_TESTS) $(COMPILED_PRINT_TESTS) $(COMPILED_MOD_BUILTIN_TESTS) $(COMPILED_SECP_CAIRO0_HINTS) $(COMPILED_KZG_DA_CAIRO0_HINTS) $(COMPILED_SEGMENT_ARENA_CAIRO0_HINTS) cairo_proof_programs: $(COMPILED_PROOF_TESTS) $(COMPILED_MOD_BUILTIN_PROOF_TESTS) $(COMPILED_STWO_EXCLUSIVE_TESTS) cairo_bench_programs: $(COMPILED_BENCHES) @@ -286,7 +295,7 @@ ifdef TEST_COLLECT_COVERAGE TEST_COMMAND:=cargo llvm-cov nextest --no-report endif -test: cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_2_test_contracts cairo_1_program +test: cairo_proof_programs cairo_test_programs cairo_test_suite_programs cairo_1_test_contracts cairo_2_test_contracts cairo_1_program $(TEST_COMMAND) --workspace --features "test_utils, cairo-1-hints" test-extensive_hints: cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_1_program cairo_2_test_contracts $(TEST_COMMAND) --workspace --features "test_utils, cairo-1-hints, cairo-0-secp-hints, cairo-0-data-availability-hints, extensive_hints" From 828320a88f5e556983b66dc3c4032fa52c5c9e45 Mon Sep 17 00:00:00 2001 From: "Naor.d" Date: Sun, 29 Mar 2026 17:19:03 +0300 Subject: [PATCH 2/4] chore: update CHANGELOG for PR #2380 Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c26e7c2eb..32800e5a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ Both branches support Stwo prover opcodes (Blake2s, QM31) since v2.0.0. #### Upcoming Changes * feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2381](https://github.com/starkware-libs/cairo-vm/pull/2381) +* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2378](https://github.com/starkware-libs/cairo-vm/pull/2378) +* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `function_runner` feature flag [#2378](https://github.com/starkware-libs/cairo-vm/pull/2378) +* feat(makefile,ci): add `cairo_test_suite_programs` Makefile target and CI integration to compile Cairo test suite programs before running tests [#2380](https://github.com/starkware-libs/cairo-vm/pull/2380) * Add Stwo cairo runner API [#2351](https://github.com/lambdaclass/cairo-vm/pull/2351) From d8f7f68d2df6fda83a77fa4df16f7547018e404b Mon Sep 17 00:00:00 2001 From: "Naor.d" Date: Mon, 6 Apr 2026 23:28:09 +0300 Subject: [PATCH 3/4] update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32800e5a5a..267072dc63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Both branches support Stwo prover opcodes (Blake2s, QM31) since v2.0.0. #### Upcoming Changes * feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2381](https://github.com/starkware-libs/cairo-vm/pull/2381) * feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2378](https://github.com/starkware-libs/cairo-vm/pull/2378) -* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `function_runner` feature flag [#2378](https://github.com/starkware-libs/cairo-vm/pull/2378) + * feat(makefile,ci): add `cairo_test_suite_programs` Makefile target and CI integration to compile Cairo test suite programs before running tests [#2380](https://github.com/starkware-libs/cairo-vm/pull/2380) * Add Stwo cairo runner API [#2351](https://github.com/lambdaclass/cairo-vm/pull/2351) From f0bb2d03a370bbb2241876d55ea3f99f57388592 Mon Sep 17 00:00:00 2001 From: "Naor.d" Date: Mon, 6 Apr 2026 23:43:49 +0300 Subject: [PATCH 4/4] fix(ci): remove stale function_runner feature flag from coverage command Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/rust.yml | 2 +- CHANGELOG.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 52210d6d11..8ba9fd8c53 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -337,7 +337,7 @@ jobs: export PARTITION=${MATRIX_TARGET#*#} cargo llvm-cov nextest --lcov --output-path lcov-${{ matrix.target }}-${{ matrix.special_features }}.info \ --partition count:${PARTITION}/4 \ - --workspace --features "cairo-1-hints, test_utils, function_runner, ${{ matrix.special_features }}" + --workspace --features "cairo-1-hints, test_utils, ${{ matrix.special_features }}" - name: Save coverage uses: actions/cache/save@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 267072dc63..25bcc8ac3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,7 @@ Both branches support Stwo prover opcodes (Blake2s, QM31) since v2.0.0. --- #### Upcoming Changes -* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2381](https://github.com/starkware-libs/cairo-vm/pull/2381) -* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq!`, `load_cairo_program!` macros and `expect_*` error checkers, behind `test_utils` feature flag [#2378](https://github.com/starkware-libs/cairo-vm/pull/2378) +* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq`, `load_cairo_program!` macro and `expect_*` error checkers, behind `test_utils` feature flag [#2381](https://github.com/starkware-libs/cairo-vm/pull/2381) * feat(makefile,ci): add `cairo_test_suite_programs` Makefile target and CI integration to compile Cairo test suite programs before running tests [#2380](https://github.com/starkware-libs/cairo-vm/pull/2380)