feat: cherry-pick #714

Author: fakedev9999

book/fault_proofs/proposer.md

@@ -78,6 +78,9 @@ Depending on the one you choose, you must provide the corresponding environment
 
 | Variable | Description | Default Value |
 |----------|-------------|---------------|
+| `RPC_CONCURRENCY` | Maximum number of concurrent RPC requests. Lower this value (e.g., `3`-`5`) if hitting 429 rate limit errors. | `10` |
+| `L1_CONFIG_DIR` | The directory containing the L1 chain configuration files. | `<project-root>/configs/L1` |
+| `L2_CONFIG_DIR` | Directory containing L2 chain configuration files | `<project-root>/configs/L2` |
 | `MOCK_MODE` | Whether to use mock mode | `false` |
 | `FAST_FINALITY_MODE` | Whether to use fast finality mode | `false` |
 | `RANGE_PROOF_STRATEGY` | Proof fulfillment strategy for range proofs. Set to `hosted` to use the hosted proof strategy. | `reserved` |

book/troubleshooting.md

@@ -46,3 +46,38 @@ let l1_head_number = l1_head_number + 100;
 The error occurs in the derivation pipeline when attempting to validate L2 blocks. The L1 head must be sufficiently ahead of the batch posting block to ensure all required data is available and the safe head state is consistent. The buffer of 20 blocks is added empirically to handle most cases where RPCs may have an incorrect view of the safe head state and have minimum overhead for the derivation process.
 
 Reference: [Fetcher Implementation](https://github.com/succinctlabs/op-succinct/blob/5dfc43928c75cef0ebf881d10bd8b3dcbe273419/utils/host/src/fetcher.rs#L773)
+
+### RPC Rate Limit Errors (429)
+
+**Error Message:**
+
+```text
+error code 429: Too Many Requests
+```
+
+or
+
+```text
+rate limit exceeded
+```
+
+**Cause:**
+This error occurs when your RPC provider is rate-limiting requests due to too many concurrent calls. OP Succinct makes concurrent RPC requests to fetch block data efficiently, which can exceed the limits of free or low-tier RPC plans.
+
+**Solution:**
+Set the `RPC_CONCURRENCY` environment variable to a lower value:
+
+```bash
+# For low-tier RPC plans, try 3-5 concurrent requests
+export RPC_CONCURRENCY=5
+
+# For very restrictive plans
+export RPC_CONCURRENCY=3
+```
+
+The default is `10` concurrent requests. Reducing this value will make operations slower but more reliable for rate-limited RPC endpoints.
+
+**Recommendations:**
+- Free RPC tiers: Set `RPC_CONCURRENCY=3`
+- Basic paid tiers: Set `RPC_CONCURRENCY=5`
+- Professional tiers: Default `10` should work, or increase for faster performance

book/validity/proposer.md

@@ -39,6 +39,7 @@ Before starting the proposer, ensure you have deployed the relevant contracts an
 | Parameter | Description |
 |-----------|-------------|
 | `L1_BEACON_RPC` | L1 Consensus (Beacon) Node. Could be required for integrations that access consensus-layer data. |
+| `RPC_CONCURRENCY` | Default: `10`. Maximum number of concurrent RPC requests. Lower this value (e.g., `3`-`5`) if you're hitting 429 rate limit errors from your RPC provider. |
 | `NETWORK_RPC_URL` | Default: `https://rpc.production.succinct.xyz`. RPC URL for the Succinct Prover Network. |
 | `DATABASE_URL` | Default: `postgres://op-succinct@postgres:5432/op-succinct`. The address of a Postgres database for storing the intermediate proposer state. |
 | `DGF_ADDRESS` | Address of the `DisputeGameFactory` contract. Note: If set, the proposer will create a dispute game with the DisputeGameFactory, rather than the `OPSuccinctL2OutputOracle`. Compatible with `OptimismPortal2`. |

scripts/utils/bin/cost_estimator.rs

@@ -7,7 +7,7 @@ use op_succinct_host_utils::{
         get_rolling_block_range, get_validated_block_range, split_range_based_on_safe_heads,
         split_range_basic, SpanBatchRange,
     },
-    fetcher::OPSuccinctDataFetcher,
+    fetcher::{get_rpc_concurrency_from_env, OPSuccinctDataFetcher},
     host::OPSuccinctHost,
     stats::ExecutionStats,
     witness_generation::WitnessGenerator,
@@ -47,7 +47,7 @@ async fn execute_blocks_and_write_stats_csv<H: OPSuccinctHost>(
                 .expect("Failed to fetch block data range.");
             (range, block_data)
         })
-        .buffered(15)
+        .buffered(get_rpc_concurrency_from_env())
         .collect::<Vec<_>>()
         .await;
 
@@ -252,7 +252,7 @@ async fn main() -> Result<()> {
                 .await
                 .expect("Failed to get host CLI args")
         })
-        .buffered(15)
+        .buffered(get_rpc_concurrency_from_env())
         .collect::<Vec<_>>()
         .await;
 

scripts/utils/bin/gen_sp1_test_artifacts.rs

@@ -4,7 +4,7 @@ use futures::StreamExt;
 use log::info;
 use op_succinct_host_utils::{
     block_range::{get_validated_block_range, split_range_basic},
-    fetcher::OPSuccinctDataFetcher,
+    fetcher::{get_rpc_concurrency_from_env, OPSuccinctDataFetcher},
     host::OPSuccinctHost,
     witness_generation::WitnessGenerator,
 };
@@ -49,7 +49,7 @@ async fn main() -> Result<()> {
                 .await
                 .expect("Failed to get host CLI args")
         })
-        .buffered(15)
+        .buffered(get_rpc_concurrency_from_env())
         .collect::<Vec<_>>()
         .await;
 

utils/host/src/block_range.rs

@@ -10,7 +10,7 @@ use kona_rpc::{OutputResponse, SafeHeadResponse};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    fetcher::{OPSuccinctDataFetcher, RPCMode},
+    fetcher::{get_rpc_concurrency_from_env, OPSuccinctDataFetcher, RPCMode},
     host::OPSuccinctHost,
 };
 
@@ -154,7 +154,7 @@ pub async fn split_range_based_on_safe_heads(
                 .expect("Failed to fetch safe head");
             result.safe_head.number
         })
-        .buffered(15)
+        .buffered(get_rpc_concurrency_from_env())
         .collect::<HashSet<_>>()
         .await;
 

utils/host/src/fetcher.rs

@@ -56,6 +56,9 @@ pub struct RPCConfig {
     pub l2_rpc: Url,
     // TODO(fakedev9999): Make optional if possible.
     pub l2_node_rpc: Url,
+    /// Maximum number of concurrent RPC requests. Default: 5.
+    /// Configure via RPC_CONCURRENCY environment variable.
+    pub concurrency: usize,
 }
 
 /// The mode corresponding to the chain we are fetching data for.
@@ -67,12 +70,26 @@ pub enum RPCMode {
     L2Node,
 }
 
+/// Default RPC concurrency limit.
+/// Set RPC_CONCURRENCY env var to a lower value (e.g., 3-5) if hitting 429 rate limits.
+pub const DEFAULT_RPC_CONCURRENCY: usize = 10;
+
+/// Gets the RPC concurrency from the RPC_CONCURRENCY environment variable.
+/// Returns DEFAULT_RPC_CONCURRENCY (10) if not set or invalid.
+pub fn get_rpc_concurrency_from_env() -> usize {
+    env::var("RPC_CONCURRENCY")
+        .ok()
+        .and_then(|s| s.parse::<usize>().ok())
+        .unwrap_or(DEFAULT_RPC_CONCURRENCY)
+}
+
 /// Gets the RPC URLs from environment variables.
 ///
 /// L1_RPC: The L1 RPC URL.
 /// L1_BEACON_RPC: The L1 beacon RPC URL.
 /// L2_RPC: The L2 RPC URL.
 /// L2_NODE_RPC: The L2 node RPC URL.
+/// RPC_CONCURRENCY: Maximum concurrent RPC requests (default: 5).
 pub fn get_rpcs_from_env() -> RPCConfig {
     let l1_rpc = env::var("L1_RPC").expect("L1_RPC must be set");
     let maybe_l1_beacon_rpc = env::var("L1_BEACON_RPC").ok();
@@ -92,6 +109,7 @@ pub fn get_rpcs_from_env() -> RPCConfig {
         l1_beacon_rpc,
         l2_rpc: Url::parse(&l2_rpc).expect("L2_RPC must be a valid URL"),
         l2_node_rpc: Url::parse(&l2_node_rpc).expect("L2_NODE_RPC must be a valid URL"),
+        concurrency: get_rpc_concurrency_from_env(),
     }
 }
 
@@ -214,7 +232,7 @@ impl OPSuccinctDataFetcher {
                     total_tx_fees,
                 })
             })
-            .buffered(10)
+            .buffered(self.rpc_config.concurrency)
             .collect::<Vec<Result<BlockInfo>>>()
             .await;
 
@@ -470,7 +488,7 @@ impl OPSuccinctDataFetcher {
         // Process blocks in batches of 10, but maintain original order
         let results = stream::iter(block_numbers)
             .map(|block_number| self.get_l1_header(block_number.into()))
-            .buffered(10)
+            .buffered(self.rpc_config.concurrency)
             .collect::<Vec<_>>()
             .await;
 

validity/src/proposer.rs

@@ -286,7 +286,7 @@ where
                         self.driver_config.fetcher.clone(),
                     )
                 })
-                .buffered(10) // Do 10 at a time, otherwise it's too slow when fetching the block range data.
+                .buffered(self.driver_config.fetcher.rpc_config.concurrency)
                 .try_collect::<Vec<OPSuccinctRequest>>()
                 .await?;