apollo_deployments: test overrides list match

Author: nimrod-starkware

Cargo.lock

@@ -10,6 +10,7 @@
   "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_base_millis": 30,
   "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_max_delay_millis": 30000,
   "state_sync_config.static_config.central_sync_client_config.sync_config.base_layer_propagation_sleep_duration": 10,
+  "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_before_tip_to_disable_batching": 100,
   "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_max_stream_size": 1000,
   "state_sync_config.static_config.central_sync_client_config.sync_config.collect_pending_data": false,
   "state_sync_config.static_config.central_sync_client_config.sync_config.latest_block_poll_interval_millis": 500,

crates/apollo_deployments/resources/app_configs/replacer_state_sync_config.json

@@ -9,6 +9,7 @@
   "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_base_millis": 30,
   "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_max_delay_millis": 30000,
   "state_sync_config.static_config.central_sync_client_config.sync_config.base_layer_propagation_sleep_duration": 10,
+  "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_before_tip_to_disable_batching": 100,
   "state_sync_config.static_config.central_sync_client_config.sync_config.latest_block_poll_interval_millis": 500,
   "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_max_stream_size": 1000,
   "state_sync_config.static_config.central_sync_client_config.sync_config.collect_pending_data": false,

crates/apollo_deployments/resources/app_configs/state_sync_config.json

@@ -58,6 +58,7 @@ apollo_state_sync_types.workspace = true
 futures.workspace = true
 metrics-exporter-prometheus.workspace = true
 papyrus_base_layer.workspace = true
+serde_json.workspace = true
 tokio.workspace = true
 tokio-util = { workspace = true, optional = true, features = ["rt"] }
 tracing.workspace = true

crates/apollo_node/Cargo.toml

@@ -0,0 +1,164 @@
+//! Local preset/native config-load parity check (uncommitted dev tool).
+//!
+//! Loads two config artifacts produced by the deployment pipeline for the same node:
+//!   - a `preset` artifact (flat dotted-key) via `--config_format preset`
+//!   - a `native` artifact (nested) via `--config_format native`
+//! Both are loaded with the SAME secrets file as a second `--config_file` so secret
+//! values match by construction, then the deserialized `SequencerNodeConfig`s are
+//! compared via the derived `PartialEq`.
+//!
+//! NOTE: this uses `SequencerNodeConfig::load_and_process` (deserialize only), NOT
+//! `load_and_validate_config`. `validate_node_config` additionally checks runtime
+//! environment invariants (e.g. that `/data/*` paths exist and that component URLs / k8s
+//! service DNS resolve) that only hold inside the pod and are irrelevant to deserialization
+//! parity; running them locally fails on every config. We compare what the node deserializes.
+//!
+//! Exit codes: 0 = PARITY: PASS, 1 = configs differ (prints a JSON diff), 2 = a load/
+//! deserialize error occurred.
+
+use std::process::exit;
+
+use apollo_node_config::node_config::SequencerNodeConfig;
+use serde_json::Value;
+
+struct Args {
+    preset_file: String,
+    native_file: String,
+    secrets_file: String,
+}
+
+fn parse_args() -> Args {
+    let mut preset_file = None;
+    let mut native_file = None;
+    let mut secrets_file = None;
+
+    let mut raw_args = std::env::args().skip(1);
+    while let Some(flag) = raw_args.next() {
+        let value = match raw_args.next() {
+            Some(value) => value,
+            None => {
+                eprintln!("Missing value for argument {flag}");
+                exit(2);
+            }
+        };
+        match flag.as_str() {
+            "--preset-file" => preset_file = Some(value),
+            "--native-file" => native_file = Some(value),
+            "--secrets-file" => secrets_file = Some(value),
+            other => {
+                eprintln!("Unknown argument: {other}");
+                eprintln!(
+                    "Usage: config_parity --preset-file P --native-file N --secrets-file S"
+                );
+                exit(2);
+            }
+        }
+    }
+
+    match (preset_file, native_file, secrets_file) {
+        (Some(preset_file), Some(native_file), Some(secrets_file)) => {
+            Args { preset_file, native_file, secrets_file }
+        }
+        _ => {
+            eprintln!(
+                "Usage: config_parity --preset-file P --native-file N --secrets-file S"
+            );
+            exit(2);
+        }
+    }
+}
+
+fn load_preset(preset_file: &str, secrets_file: &str) -> SequencerNodeConfig {
+    let args = vec![
+        "config_parity",
+        "--config_format",
+        "preset",
+        "--config_file",
+        preset_file,
+        "--config_file",
+        secrets_file,
+    ]
+    .into_iter()
+    .map(String::from)
+    .collect();
+    match SequencerNodeConfig::load_and_process(args) {
+        Ok(config) => config,
+        Err(error) => {
+            eprintln!("Failed to load PRESET config from {preset_file}: {error}");
+            exit(2);
+        }
+    }
+}
+
+fn load_native(native_file: &str, secrets_file: &str) -> SequencerNodeConfig {
+    // Native requires the first `--config_file` to be the nested base; later files are
+    // flat secret overrides.
+    let args = vec![
+        "config_parity",
+        "--config_format",
+        "native",
+        "--config_file",
+        native_file,
+        "--config_file",
+        secrets_file,
+    ]
+    .into_iter()
+    .map(String::from)
+    .collect();
+    match SequencerNodeConfig::load_and_process(args) {
+        Ok(config) => config,
+        Err(error) => {
+            eprintln!("Failed to load NATIVE config from {native_file}: {error}");
+            exit(2);
+        }
+    }
+}
+
+/// Prints the top-level keys whose JSON values differ between the two configs.
+fn print_diff(preset_config: &SequencerNodeConfig, native_config: &SequencerNodeConfig) {
+    let preset_value = serde_json::to_value(preset_config)
+        .expect("SequencerNodeConfig should serialize to JSON");
+    let native_value = serde_json::to_value(native_config)
+        .expect("SequencerNodeConfig should serialize to JSON");
+
+    let empty = serde_json::Map::new();
+    let preset_object = preset_value.as_object().unwrap_or(&empty);
+    let native_object = native_value.as_object().unwrap_or(&empty);
+
+    let mut all_keys: Vec<&String> =
+        preset_object.keys().chain(native_object.keys()).collect();
+    all_keys.sort_unstable();
+    all_keys.dedup();
+
+    let null = Value::Null;
+    for key in all_keys {
+        let preset_field = preset_object.get(key).unwrap_or(&null);
+        let native_field = native_object.get(key).unwrap_or(&null);
+        if preset_field != native_field {
+            eprintln!("--- DIFF in top-level field `{key}` ---");
+            eprintln!(
+                "  preset: {}",
+                serde_json::to_string(preset_field).unwrap_or_default()
+            );
+            eprintln!(
+                "  native: {}",
+                serde_json::to_string(native_field).unwrap_or_default()
+            );
+        }
+    }
+}
+
+fn main() {
+    let args = parse_args();
+    let preset_config = load_preset(&args.preset_file, &args.secrets_file);
+    let native_config = load_native(&args.native_file, &args.secrets_file);
+
+    if preset_config == native_config {
+        println!("PARITY: PASS");
+        exit(0);
+    }
+
+    eprintln!("PARITY: FAIL - loaded SequencerNodeConfig values differ");
+    print_diff(&preset_config, &native_config);
+    exit(1);
+}

crates/apollo_node/src/bin/config_parity.rs

@@ -19,17 +19,70 @@ use apollo_storage::state_commitment_infos::StateCommitmentInfosStorageWriter;
 use apollo_storage::StorageWriter;
 use futures::future::pending;
 use futures::never::Never;
-use serde::{Deserialize, Serialize};
+use std::fmt;
+
+use serde::de::{self, Visitor};
+use serde::{Deserialize, Deserializer, Serialize};
 use starknet_api::block::BlockNumber;
 use tracing::info;
 use validator::Validate;
 
 #[derive(Clone, Debug, Serialize, Deserialize, Validate, PartialEq)]
 pub struct RevertConfig {
+    #[serde(deserialize_with = "deserialize_revert_block_number")]
     pub revert_up_to_and_including: BlockNumber,
     pub should_revert: bool,
 }
 
+/// Deserialize `revert_up_to_and_including`, tolerating a floating-point representation of the
+/// value.
+///
+/// The default is the `u64::MAX` "never revert" sentinel. Configs assembled via jsonnet (the
+/// `native` config format) render every number as an IEEE-754 double, so `u64::MAX`
+/// (18446744073709551615) is emitted as the nearest double, `2^64` (18446744073709551616), which
+/// overflows `u64` and would otherwise fail to deserialize. We accept a float and saturating-cast
+/// it back to `u64`, so a config produced by jsonnet deserializes to the same `BlockNumber` as the
+/// legacy flat (`preset`) path, which carries the exact `u64::MAX` integer. Plain integer values
+/// (the common case, including real revert heights) take the `u64` branch unchanged.
+fn deserialize_revert_block_number<'de, D>(deserializer: D) -> Result<BlockNumber, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    struct RevertBlockNumberVisitor;
+
+    impl<'de> Visitor<'de> for RevertBlockNumberVisitor {
+        type Value = BlockNumber;
+
+        fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+            formatter.write_str(
+                "a block number; jsonnet renders the u64::MAX sentinel as the 2^64 value, which \
+                 deserializers may present as u128 or f64",
+            )
+        }
+
+        fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
+            Ok(BlockNumber(value))
+        }
+
+        // jsonnet's f64 rounding turns u64::MAX into 2^64, which serde_json surfaces as a u128.
+        // Saturate anything past u64::MAX back to the sentinel.
+        fn visit_u128<E: de::Error>(self, value: u128) -> Result<Self::Value, E> {
+            Ok(BlockNumber(u64::try_from(value).unwrap_or(u64::MAX)))
+        }
+
+        fn visit_i64<E: de::Error>(self, value: i64) -> Result<Self::Value, E> {
+            Ok(BlockNumber(u64::try_from(value).unwrap_or(0)))
+        }
+
+        // Saturating float-to-int cast: 2^64 (and larger) maps to u64::MAX.
+        fn visit_f64<E: de::Error>(self, value: f64) -> Result<Self::Value, E> {
+            Ok(BlockNumber(value as u64))
+        }
+    }
+
+    deserializer.deserialize_any(RevertBlockNumberVisitor)
+}
+
 impl Default for RevertConfig {
     fn default() -> Self {
         Self {