Merge remote-tracking branch 'origin/staging' into feat/integrate-agg-mode-with-db

Author: MarcosNicolau

.github/workflows/build-and-test-go.yml

@@ -13,7 +13,7 @@ on:
       - "common/**"
       - "core/**"
       - "metrics/**"
-      - ".github/workflows/build-go.yml"
+      - ".github/workflows/build-and-test-go.yml"
 env:
   FFI_FOR_RELEASE: false
 jobs:
@@ -31,21 +31,83 @@ jobs:
       - name: foundry-toolchain
         uses: foundry-rs/foundry-toolchain@v1.2.0
 
+      - name: Cache SP1 bindings
+        id: cache-sp1
+        uses: actions/cache@v4
+        with:
+          path: operator/sp1/lib/libsp1_verifier_ffi.so
+          key: sp1-bindings-${{ runner.os }}-${{ hashFiles('operator/sp1/lib/**/*.rs', 'operator/sp1/lib/Cargo.*') }}
+
       - name: Build SP1 bindings
+        if: steps.cache-sp1.outputs.cache-hit != 'true'
         run: make build_sp1_linux
 
+      - name: Clean SP1 build artifacts
+        if: steps.cache-sp1.outputs.cache-hit != 'true'
+        run: rm -rf operator/sp1/lib/target
+
+      - name: Cache Risc Zero bindings
+        id: cache-risc-zero
+        uses: actions/cache@v4
+        with:
+          path: operator/risc_zero/lib/librisc_zero_verifier_ffi.so
+          key: risc-zero-bindings-${{ runner.os }}-${{ hashFiles('operator/risc_zero/lib/**/*.rs', 'operator/risc_zero/lib/Cargo.*') }}
+
       - name: Build Risc Zero go bindings
+        if: steps.cache-risc-zero.outputs.cache-hit != 'true'
         run: make build_risc_zero_linux
 
+      - name: Clean Risc Zero build artifacts
+        if: steps.cache-risc-zero.outputs.cache-hit != 'true'
+        run: rm -rf operator/risc_zero/lib/target
+
+      - name: Cache Merkle Tree bindings
+        id: cache-merkle-tree
+        uses: actions/cache@v4
+        with:
+          path: |
+            operator/merkle_tree/lib/libmerkle_tree.so
+            operator/merkle_tree/lib/libmerkle_tree.a
+          key: merkle-tree-bindings-${{ runner.os }}-${{ hashFiles('operator/merkle_tree/lib/**/*.rs', 'operator/merkle_tree/lib/Cargo.*') }}
+
       - name: Build Merkle Tree bindings
+        if: steps.cache-merkle-tree.outputs.cache-hit != 'true'
         run: make build_merkle_tree_linux
-        
+
+      - name: Clean Merkle Tree build artifacts
+        if: steps.cache-merkle-tree.outputs.cache-hit != 'true'
+        run: rm -rf operator/merkle_tree/lib/target
+
+      - name: Cache Mina bindings
+        id: cache-mina
+        uses: actions/cache@v4
+        with:
+          path: operator/mina/lib/libmina_state_verifier_ffi.so
+          key: mina-bindings-${{ runner.os }}-${{ hashFiles('operator/mina/lib/**/*.rs', 'operator/mina/lib/Cargo.*') }}
+
       - name: Build Mina bindings
+        if: steps.cache-mina.outputs.cache-hit != 'true'
         run: make build_mina_linux
-      
+
+      - name: Clean Mina build artifacts
+        if: steps.cache-mina.outputs.cache-hit != 'true'
+        run: rm -rf operator/mina/lib/target
+
+      - name: Cache Mina Account bindings
+        id: cache-mina-account
+        uses: actions/cache@v4
+        with:
+          path: operator/mina_account/lib/libmina_account_verifier_ffi.so
+          key: mina-account-bindings-${{ runner.os }}-${{ hashFiles('operator/mina_account/lib/**/*.rs', 'operator/mina_account/lib/Cargo.*') }}
+
       - name: Build Mina Account bindings
+        if: steps.cache-mina-account.outputs.cache-hit != 'true'
         run: make build_mina_account_linux
-      
+
+      - name: Clean Mina Account build artifacts
+        if: steps.cache-mina-account.outputs.cache-hit != 'true'
+        run: rm -rf operator/mina_account/lib/target
+
       - name: Build operator
         run: go build operator/cmd/main.go
 

Makefile

@@ -252,6 +252,9 @@ proof_aggregator_start_dev: is_aggregator_set reset_last_aggregated_block ./aggr
 proof_aggregator_start_dev_ethereum_package: is_aggregator_set reset_last_aggregated_block ./aggregation_mode/target/release/proof_aggregator_dev ## Starts proof aggregator with mock proofs (DEV mode) in ethereum package. Parameters: AGGREGATOR=<sp1|risc0>
 	AGGREGATOR=$(AGGREGATOR) RISC0_DEV_MODE=1 ./aggregation_mode/target/release/proof_aggregator_dev config-files/config-proof-aggregator-mock-ethereum-package.yaml
 
+proof_aggregator_test_without_compiling_agg_programs:
+	cd aggregation_mode && SKIP_AGG_PROGRAMS_BUILD=1 cargo test -p proof_aggregator --tests -- --nocapture
+
 ### All CPU proof aggregator receipts
 ./aggregation_mode/target/release/proof_aggregator_cpu: $(AGGREGATION_MODE_SOURCES)
 	AGGREGATOR=$(AGGREGATOR) cargo build --features prove --manifest-path ./aggregation_mode/Cargo.toml --release --bin proof_aggregator_cpu
@@ -320,6 +323,22 @@ agg_mode_batcher_start_local: agg_mode_run_migrations
 agg_mode_batcher_start_ethereum_package: agg_mode_run_migrations
 	cargo run --manifest-path ./aggregation_mode/Cargo.toml --release --bin agg_mode_batcher -- config-files/config-agg-mode-batcher-ethereum-package.yaml
 
+AGG_MODE_SENDER ?= 0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+agg_mode_batcher_send_payment:
+	@cast send --value 1ether \
+		0x922D6956C99E12DFeB3224DEA977D0939758A1Fe \
+		--private-key 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+
+agg_mode_batcher_send_sp1_proof:
+	@NONCE=$$(curl -s http://127.0.0.1:8089/nonce/0x70997970C51812dc3A010C7d01b50e0d17dc79C8 | jq -r '.data.nonce'); \
+	curl -X POST \
+		-H "Content-Type: multipart/form-data" \
+		-F "nonce=$${NONCE}" \
+		-F "proof=@scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof" \
+		-F "program_vk=@scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin" \
+		-F "signature_hex=0x0" \
+		http://127.0.0.1:8089/proof/sp1
+
 __AGGREGATOR__: ## ____
 
 aggregator_start: ## Start the Aggregator. Parameters: ENVIRONMENT=<devnet|testnet|mainnet>, AGG_CONFIG_FILE

aggregation_mode/Cargo.lock

@@ -8,9 +8,12 @@ serde =  { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 aligned-sdk = { workspace = true }
+sp1-sdk = { workspace = true }
 tracing = { version = "0.1", features = ["log"] }
 tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
+bincode = "1.3.3"
 actix-web = "4"
+actix-multipart = "0.7.2"
 alloy = { workspace = true }
 tokio = { version = "1", features = ["time"]}
 # TODO: enable tls

aggregation_mode/batcher/Cargo.toml

@@ -2,3 +2,4 @@ pub mod config;
 pub mod db;
 pub mod payments;
 pub mod server;
+mod verifiers;

aggregation_mode/batcher/src/lib.rs

@@ -3,11 +3,13 @@ use std::{
     time::{SystemTime, UNIX_EPOCH},
 };
 
+use actix_multipart::form::MultipartForm;
 use actix_web::{
     web::{self, Data},
     App, HttpRequest, HttpResponse, HttpServer, Responder,
 };
 use aligned_sdk::aggregation_layer::AggregationModeProvingSystem;
+use sp1_sdk::{SP1ProofWithPublicValues, SP1VerifyingKey};
 use sqlx::types::BigDecimal;
 
 use super::{
@@ -18,9 +20,8 @@ use super::{
 use crate::{
     config::Config,
     db::Db,
-    server::types::{
-        SubmitProofRequest, SubmitProofRequestMessageRisc0, SubmitProofRequestMessageSP1,
-    },
+    server::types::{SubmitProofRequestRisc0, SubmitProofRequestSP1},
+    verifiers::{verify_sp1_proof, VerificationError},
 };
 
 #[derive(Clone, Debug)]
@@ -64,7 +65,6 @@ impl BatcherServer {
         };
 
         // TODO: validate valid ethereum address
-
         let Some(state) = req.app_data::<Data<BatcherServer>>() else {
             return HttpResponse::InternalServerError()
                 .json(AppResponse::new_unsucessfull("Internal server error", 500));
@@ -84,11 +84,8 @@ impl BatcherServer {
 
     async fn post_proof_sp1(
         req: HttpRequest,
-        body: web::Json<SubmitProofRequest<SubmitProofRequestMessageSP1>>,
+        MultipartForm(data): MultipartForm<SubmitProofRequestSP1>,
     ) -> impl Responder {
-        let data = body.into_inner();
-
-        // TODO: validate signature
         let recovered_address = "0x70997970C51812dc3A010C7d01b50e0d17dc79C8".to_lowercase();
 
         let Some(state) = req.app_data::<Data<BatcherServer>>() else {
@@ -102,7 +99,7 @@ impl BatcherServer {
                 .json(AppResponse::new_unsucessfull("Internal server error", 500));
         };
 
-        if data.nonce != (count as u64) {
+        if data.nonce.0 != (count as u64) {
             return HttpResponse::BadRequest().json(AppResponse::new_unsucessfull(
                 &format!("Invalid nonce, expected nonce = {count}"),
                 400,
@@ -140,15 +137,42 @@ impl BatcherServer {
             ));
         }
 
-        // TODO: decode proof and validate it
+        let Ok(proof_content) = tokio::fs::read(data.proof.file.path()).await else {
+            return HttpResponse::InternalServerError()
+                .json(AppResponse::new_unsucessfull("Internal server error", 500));
+        };
+
+        let Ok(proof) = bincode::deserialize::<SP1ProofWithPublicValues>(&proof_content) else {
+            return HttpResponse::BadRequest()
+                .json(AppResponse::new_unsucessfull("Invalid SP1 proof", 400));
+        };
+
+        let Ok(vk_content) = tokio::fs::read(data.program_vk.file.path()).await else {
+            return HttpResponse::InternalServerError()
+                .json(AppResponse::new_unsucessfull("Internal server error", 500));
+        };
+
+        let Ok(vk) = bincode::deserialize::<SP1VerifyingKey>(&vk_content) else {
+            return HttpResponse::BadRequest()
+                .json(AppResponse::new_unsucessfull("Invalid vk", 400));
+        };
+
+        if let Err(e) = verify_sp1_proof(&proof, &vk) {
+            let message = match e {
+                VerificationError::InvalidProof => "Proof verification failed",
+                VerificationError::UnsupportedProof => "Unsupported proof",
+            };
+
+            return HttpResponse::BadRequest().json(AppResponse::new_unsucessfull(message, 400));
+        };
 
         match state
             .db
             .insert_task(
                 &recovered_address,
                 AggregationModeProvingSystem::SP1.as_u16() as i32,
-                &data.message.proof,
-                &data.message.program_vk_commitment,
+                &proof_content,
+                &vk_content,
                 None,
             )
             .await
@@ -164,7 +188,7 @@ impl BatcherServer {
     /// TODO: complete for risc0 (see `post_proof_sp1`)
     async fn post_proof_risc0(
         _req: HttpRequest,
-        _body: web::Json<SubmitProofRequest<SubmitProofRequestMessageRisc0>>,
+        MultipartForm(_): MultipartForm<SubmitProofRequestRisc0>,
     ) -> impl Responder {
         HttpResponse::Ok().json(AppResponse::new_sucessfull(serde_json::json!({})))
     }