refactor: integration test suite (#389)

* refactor: integration test suite

start height

test

test

test

test

test

test

test

adding integration test to ci

short circuit

review fixes

* all

* clippy stuff

* warn log

* docs

Author: distractedm1nd

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
       - uses: taiki-e/install-action@nextest
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Expose github actions runtime
         uses: crazy-max/ghaction-github-runtime@v1
@@ -127,6 +127,7 @@ jobs:
     env:
       SP1_PROVER: mock
       SP1_SKIP_PROGRAM_BUILD: true
+      COVERAGE_TEST: true
     name: coverage
     steps:
       - name: Checkout sources
@@ -140,9 +141,66 @@ jobs:
       - uses: taiki-e/install-action@cargo-llvm-cov
       - uses: taiki-e/install-action@nextest
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Expose github actions runtime
+        uses: crazy-max/ghaction-github-runtime@v1
+
+      - name: Build the docker-compose stack
+        run: |
+          cat > ci/cache.json <<EOF
+          {
+            "target": {
+              "validator": {
+                "cache-from": ["type=gha,scope=validator"],
+                "cache-to": ["type=gha,mode=max,scope=validator"],
+                "output": ["type=docker"]
+              },
+              "bridge-0": {
+                "cache-from": ["type=gha,scope=bridge-0"],
+                "cache-to": ["type=gha,mode=max,scope=bridge-0"],
+                "output": ["type=docker"]
+              },
+              "bridge-1": {
+                "cache-from": ["type=gha,scope=bridge-1"],
+                "cache-to": ["type=gha,mode=max,scope=bridge-1"],
+                "output": ["type=docker"]
+              },
+              "light-0": {
+                "cache-from": ["type=gha,scope=light-0"],
+                "cache-to": ["type=gha,mode=max,scope=light-0"],
+                "output": ["type=docker"]
+              }
+            }
+          }
+          EOF
+          cd ci && docker buildx bake --file docker-compose.yml --file cache.json --load
+
+      - name: Run the docker-compose stack
+        run: docker compose -f ci/docker-compose.yml up --no-build -d
+
+      - name: Wait for bridge node 0 to start
+        run: |
+          while ! docker compose -f ci/docker-compose.yml logs bridge-0 | grep -q 'Configuration finished. Running a bridge node'; do
+            sleep 1
+          done
+
+      - name: Wait for bridge node 1 to start
+        run: |
+          while ! docker compose -f ci/docker-compose.yml logs bridge-1 | grep -q 'Configuration finished. Running a bridge node'; do
+            sleep 1
+          done
+
+      - name: Wait for light node 0 to start
+        run: |
+          while ! docker compose -f ci/docker-compose.yml logs light-0 | grep -q 'Configuration finished. Running a light node'; do
+            sleep 1
+          done
+
       - name: Collect unit-test coverage data
         run: |
-          cargo llvm-cov --no-report nextest --lib --release -- --skip test_light_client_prover_talking
+          cargo llvm-cov --no-report nextest --lib --release
           cargo llvm-cov report --release --codecov --output-path codecov.info
 
       - name: Upload coverage data to codecov

Cargo.lock

@@ -164,6 +164,7 @@ mod tests {
     fn test_light_client_da_args_application() -> Result<()> {
         // Test light client DA args
         let mut light_client_config = LightClientDAConfig::Celestia(CelestiaLightClientDAConfig {
+            bootnodes: vec![],
             celestia_network: CelestiaNetwork::Arabica,
             snark_namespace_id: "old_namespace".to_string(),
             fetch_timeout: Duration::from_secs(30),
@@ -207,6 +208,7 @@ mod tests {
     #[test]
     fn test_light_client_da_args_partial_override() -> Result<()> {
         let mut light_client_config = LightClientDAConfig::Celestia(CelestiaLightClientDAConfig {
+            bootnodes: vec![],
             celestia_network: CelestiaNetwork::Arabica,
             snark_namespace_id: "old_namespace".to_string(),
             fetch_timeout: Duration::from_secs(30),
@@ -317,6 +319,7 @@ mod tests {
     fn test_no_da_type_specified() -> Result<()> {
         // When no DA type is specified in CLI args, config should remain unchanged
         let original_config = LightClientDAConfig::Celestia(CelestiaLightClientDAConfig {
+            bootnodes: vec![],
             celestia_network: CelestiaNetwork::Arabica,
             snark_namespace_id: "original_namespace".to_string(),
             fetch_timeout: Duration::from_secs(30),
@@ -368,6 +371,7 @@ mod tests {
     #[test]
     fn test_celestia_store_type_disk_without_path_error() {
         let mut config = LightClientDAConfig::Celestia(CelestiaLightClientDAConfig {
+            bootnodes: vec![],
             celestia_network: CelestiaNetwork::Arabica,
             snark_namespace_id: "namespace".to_string(),
             fetch_timeout: Duration::from_secs(30),

crates/cli/src/apply_args/da.rs

@@ -32,6 +32,7 @@ prism-events = { workspace = true }
 prism-presets = { workspace = true }
 mockall = { workspace = true }
 bincode = { workspace = true }
+libp2p = { workspace = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 dirs = { workspace = true }

crates/da/Cargo.toml

@@ -1,17 +1,19 @@
 use anyhow::{Result, anyhow};
 use async_trait::async_trait;
 use celestia_types::nmt::Namespace;
+use libp2p::Multiaddr;
 use lumina_node::{Node, NodeBuilder};
-#[cfg(target_arch = "wasm32")]
-use lumina_node::{blockstore::IndexedDbBlockstore, store::IndexedDbStore};
 use prism_errors::DataAvailabilityError;
 use prism_events::{EventChannel, EventPublisher};
+use std::{self, str::FromStr, sync::Arc, time::Duration};
+use tokio::sync::{Mutex, RwLock};
+use tracing::{trace, warn};
+
+#[cfg(target_arch = "wasm32")]
+use lumina_node::{blockstore::IndexedDbBlockstore, store::IndexedDbStore};
 use prism_presets::{ApplyPreset, LightClientPreset, PresetError};
 use serde::{Deserialize, Serialize};
 use serde_with::{DurationSeconds, serde_as};
-use std::{self, sync::Arc, time::Duration};
-use tokio::sync::{Mutex, RwLock};
-use tracing::{trace, warn};
 #[cfg(not(target_arch = "wasm32"))]
 use {
     blockstore::EitherBlockstore,
@@ -63,6 +65,15 @@ pub struct CelestiaLightClientDAConfig {
     /// Different networks have different block times and fee structures.
     pub celestia_network: CelestiaNetwork,
 
+    /// List of bootnodes to connect to.
+    ///
+    /// Bootnodes are used to bootstrap the connection to the network.
+    /// They are not required for normal operation but can help with initial
+    /// connection in case the network is not fully connected or a custom network are used.
+    ///
+    /// Are the String representations of libp2p multiaddresses on the Celestia network.
+    pub bootnodes: Vec<String>,
+
     /// Hex-encoded namespace ID for SNARK proofs.
     ///
     /// Light clients will only download and verify data from this namespace,
@@ -204,6 +215,7 @@ impl Default for CelestiaLightClientDAConfig {
         Self {
             celestia_network: CelestiaNetwork::Arabica,
             snark_namespace_id: "00000000000000de1008".to_string(),
+            bootnodes: Vec::new(),
             pruning_window: DEFAULT_PRUNING_WINDOW,
             fetch_timeout: DEFAULT_FETCH_TIMEOUT,
             fetch_max_retries: DEFAULT_FETCH_MAX_RETRIES,
@@ -342,11 +354,31 @@ impl LightClientConnection {
     pub async fn new(config: &CelestiaLightClientDAConfig) -> Result<Self, DataAvailabilityError> {
         let (blockstore, store) = Self::setup_stores(&config.store).await?;
 
-        let (node, event_subscriber) = NodeBuilder::new()
+        let mut node = NodeBuilder::new()
             .network(config.celestia_network.clone())
             .store(store)
             .blockstore(blockstore)
-            .pruning_window(config.pruning_window)
+            .pruning_window(config.pruning_window);
+
+        if !config.bootnodes.is_empty() {
+            let multiaddrs: Vec<Multiaddr> = config
+                .bootnodes
+                .clone()
+                .into_iter()
+                .filter_map(|addr| Multiaddr::from_str(&addr).ok())
+                .collect();
+
+            if multiaddrs.len() != config.bootnodes.len() {
+                warn!(
+                    "Some bootnodes failed to parse to libp2p multiaddrs. Valid addresses contain: {:#?}",
+                    multiaddrs
+                );
+            }
+
+            node = node.bootnodes(multiaddrs);
+        }
+
+        let (node, event_subscriber) = node
             .start_subscribed()
             .await
             .map_err(|e| DataAvailabilityError::InitializationError(e.to_string()))?;

crates/da/src/celestia/light_client.rs

@@ -92,6 +92,8 @@ pub struct LightClient {
 
     // The latest commitment.
     latest_commitment: Arc<RwLock<Option<Digest>>>,
+
+    mock_proof_verification: bool,
 }
 
 #[derive(Default, Clone)]
@@ -126,9 +128,25 @@ impl LightClient {
             latest_commitment: Arc::new(RwLock::new(None)),
             sync_state,
             cancellation_token,
+            mock_proof_verification: false,
         }
     }
 
+    /// Enables mock proof verification for testing purposes only.
+    ///
+    /// # Safety
+    /// This method disables cryptographic proof verification, which is a critical
+    /// security mechanism. It should ONLY be used in test environments and NEVER
+    /// in production code.
+    ///
+    /// # Usage
+    /// This is intended for integration tests where proof generation/verification
+    /// may be too slow or unavailable.
+    pub fn enable_mock_proof_verification(&mut self) {
+        error!("PROOF VERIFICATION IS DISABLED - FOR TESTING ONLY");
+        self.mock_proof_verification = true;
+    }
+
     pub async fn get_sync_state(&self) -> SyncState {
         self.sync_state.read().await.clone()
     }
@@ -396,7 +414,10 @@ impl LightClient {
                     height: epoch.height(),
                     error: e.to_string(),
                 });
-                return Err(anyhow::anyhow!(e));
+                if !self.mock_proof_verification {
+                    return Err(anyhow::anyhow!(e));
+                }
+                epoch.commitments()
             }
         };
         let curr_commitment = commitments.current;

crates/node_types/lightclient/src/lightclient.rs

@@ -20,6 +20,7 @@ prism-storage.workspace = true
 prism-prover = { workspace = true }
 prism-keys = { workspace = true }
 prism-lightclient = { workspace = true }
+celestia-rpc = { workspace = true }
 tokio-util = { workspace = true }
 prism-da.workspace = true
 rand.workspace = true