apollo_node: add new CLI arg to parse config

Author: nimrod-starkware

crates/apollo_config/Cargo.toml

@@ -8,7 +8,7 @@ description = "A library for handling node configuration."
 
 [dependencies]
 apollo_infra_utils.workspace = true
-clap = { workspace = true, features = ["env", "string"] }
+clap = { workspace = true, features = ["derive", "env", "string"] }
 colored = { workspace = true, optional = true }
 const_format.workspace = true
 itertools.workspace = true

crates/apollo_config/src/command.rs

@@ -7,11 +7,13 @@ use serde_json::{json, Value};
 use crate::loading::update_config_map;
 use crate::{
     ConfigError,
+    ConfigFormat,
     ParamPath,
     SerializationType,
     SerializedParam,
     CONFIG_FILE_ARG_NAME,
     CONFIG_FILE_SHORT_ARG_NAME,
+    CONFIG_FORMAT_ARG_NAME,
 };
 
 pub(crate) fn get_command_matches(
@@ -51,6 +53,16 @@ fn build_args_parser(config_map: &BTreeMap<ParamPath, SerializedParam>) -> Vec<A
             .value_parser(value_parser!(PathBuf))
             .num_args(1..) // Allow multiple values
             .action(clap::ArgAction::Append), // Collect multiple occurrences
+        // How the --config_file arguments are interpreted. Absent => ConfigFormat::Preset.
+        Arg::new(CONFIG_FORMAT_ARG_NAME)
+            .long(CONFIG_FORMAT_ARG_NAME)
+            .num_args(1)
+            .value_parser(value_parser!(ConfigFormat))
+            .help(
+                "How the --config_file arguments are interpreted: 'preset' (flat dotted-key, \
+                 layered) or 'native' (the first file is nested serde, later files are flat \
+                 secret overrides)",
+            ),
     ];
 
     for (param_path, serialized_param) in config_map.iter() {

crates/apollo_config/src/config_test.rs

@@ -647,6 +647,215 @@ fn test_load_many_custom_config_files() {
     assert_eq!(param_path, "custom value");
 }
 
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeInnerConfig {
+    a: usize,
+    b: String,
+}
+
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeConfig {
+    top: String,
+    inner: NativeInnerConfig,
+}
+
+// Mirrors the real secrets layout: dotted keys reach deep, real, nested leaves (e.g.
+// `consensus_manager_config.network_config.secret_key`), never single-token pointer targets.
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeNetworkConfig {
+    secret_key: String,
+    other_network_field: bool,
+}
+
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeComponentConfig {
+    network_config: NativeNetworkConfig,
+    other_component_field: u32,
+}
+
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeDeepConfig {
+    component_config: NativeComponentConfig,
+    other_top_field: String,
+}
+
+// A `None` optional sub-config, mirroring a disabled component (e.g. `central_sync_client_config`).
+#[derive(Deserialize, Debug, PartialEq)]
+struct NativeOptionalConfig {
+    optional_component: Option<NativeComponentConfig>,
+    top_field: String,
+}
+
+// Loads a config in `--config_format native` mode. The schema file is unused by the native path but
+// still parsed, so an empty object is supplied as a valid (empty) schema.
+fn load_native_config<T: for<'a> Deserialize<'a>>(
+    config_file_args: Vec<&str>,
+) -> Result<T, ConfigError> {
+    let schema_file = NamedTempFile::new().unwrap();
+    std::fs::write(schema_file.path(), json!({}).to_string()).unwrap();
+
+    let args = chain!(["Testing", "--config_format", "native"], config_file_args)
+        .map(|arg| arg.to_owned())
+        .collect();
+
+    load_and_process_config::<T>(
+        File::open(schema_file.path()).unwrap(),
+        Command::new("Program"),
+        args,
+        false,
+    )
+}
+
+#[test]
+fn test_native_config_with_secret_overrides() {
+    let base_file = NamedTempFile::new().unwrap();
+    let base_config = json!({"top": "base_top", "inner": {"a": 1, "b": "base_b"}});
+    std::fs::write(base_file.path(), base_config.to_string()).unwrap();
+    // Secrets stay flat dotted-key and are applied onto the nested base.
+    let secret_file = NamedTempFile::new().unwrap();
+    std::fs::write(secret_file.path(), json!({"inner.b": "secret_b"}).to_string()).unwrap();
+
+    let loaded: NativeConfig = load_native_config(vec![
+        CONFIG_FILE_ARG,
+        base_file.path().to_str().unwrap(),
+        CONFIG_FILE_ARG,
+        secret_file.path().to_str().unwrap(),
+    ])
+    .unwrap();
+
+    assert_eq!(
+        loaded,
+        NativeConfig {
+            top: "base_top".to_owned(),
+            inner: NativeInnerConfig { a: 1, b: "secret_b".to_owned() },
+        }
+    );
+}
+
+#[test]
+fn test_native_config_single_file() {
+    let base_file = NamedTempFile::new().unwrap();
+    let base_config = json!({"top": "base_top", "inner": {"a": 1, "b": "base_b"}});
+    std::fs::write(base_file.path(), base_config.to_string()).unwrap();
+
+    let loaded: NativeConfig =
+        load_native_config(vec![CONFIG_FILE_ARG, base_file.path().to_str().unwrap()]).unwrap();
+
+    assert_eq!(
+        loaded,
+        NativeConfig {
+            top: "base_top".to_owned(),
+            inner: NativeInnerConfig { a: 1, b: "base_b".to_owned() },
+        }
+    );
+}
+
+#[test]
+fn test_native_config_without_config_file() {
+    let result = load_native_config::<NativeConfig>(vec![]);
+    assert_matches!(result, Err(ConfigError::NativeModeRequiresConfigFile));
+}
+
+// Mirrors the production secrets file (crates/apollo_deployments/resources/testing_secrets.json):
+// each secret is a flat dotted key naming a deep, real nested leaf. Asserts the override lands at
+// exactly that leaf and every sibling at every level is preserved.
+#[test]
+fn test_native_config_deep_secret_override_preserves_siblings() {
+    let base_file = NamedTempFile::new().unwrap();
+    let base_config = json!({
+        "other_top_field": "base_top",
+        "component_config": {
+            "other_component_field": 7,
+            "network_config": {
+                "secret_key": "base_secret",
+                "other_network_field": true
+            }
+        }
+    });
+    std::fs::write(base_file.path(), base_config.to_string()).unwrap();
+
+    let secret_file = NamedTempFile::new().unwrap();
+    let secret_config = json!({"component_config.network_config.secret_key": "0xabc"});
+    std::fs::write(secret_file.path(), secret_config.to_string()).unwrap();
+
+    let loaded: NativeDeepConfig = load_native_config(vec![
+        CONFIG_FILE_ARG,
+        base_file.path().to_str().unwrap(),
+        CONFIG_FILE_ARG,
+        secret_file.path().to_str().unwrap(),
+    ])
+    .unwrap();
+
+    assert_eq!(
+        loaded,
+        NativeDeepConfig {
+            other_top_field: "base_top".to_owned(),
+            component_config: NativeComponentConfig {
+                other_component_field: 7,
+                network_config: NativeNetworkConfig {
+                    secret_key: "0xabc".to_owned(),
+                    other_network_field: true,
+                },
+            },
+        }
+    );
+}
+
+// A secret aimed at a child of a `None` (null) intermediate must be skipped, not vivified into a
+// partial object — otherwise deserialization of the (incomplete) sub-config would fail. The
+// disabled component stays `None`.
+#[test]
+fn test_native_config_skips_secret_under_none_intermediate() {
+    let base_file = NamedTempFile::new().unwrap();
+    let base_config = json!({"optional_component": null, "top_field": "base"});
+    std::fs::write(base_file.path(), base_config.to_string()).unwrap();
+
+    let secret_file = NamedTempFile::new().unwrap();
+    let secret_config = json!({"optional_component.network_config.secret_key": "0xabc"});
+    std::fs::write(secret_file.path(), secret_config.to_string()).unwrap();
+
+    let loaded: NativeOptionalConfig = load_native_config(vec![
+        CONFIG_FILE_ARG,
+        base_file.path().to_str().unwrap(),
+        CONFIG_FILE_ARG,
+        secret_file.path().to_str().unwrap(),
+    ])
+    .unwrap();
+
+    assert_eq!(
+        loaded,
+        NativeOptionalConfig { optional_component: None, top_field: "base".to_owned() }
+    );
+}
+
+// A secret naming a leaf that is absent from an existing parent object is created (the parent
+// exists, so the component is enabled and the override is relevant).
+#[test]
+fn test_native_config_creates_absent_leaf_in_existing_parent() {
+    let base_file = NamedTempFile::new().unwrap();
+    let base_config = json!({"top": "base_top", "inner": {"a": 1}});
+    std::fs::write(base_file.path(), base_config.to_string()).unwrap();
+
+    let secret_file = NamedTempFile::new().unwrap();
+    std::fs::write(secret_file.path(), json!({"inner.b": "created_b"}).to_string()).unwrap();
+
+    let loaded: NativeConfig = load_native_config(vec![
+        CONFIG_FILE_ARG,
+        base_file.path().to_str().unwrap(),
+        CONFIG_FILE_ARG,
+        secret_file.path().to_str().unwrap(),
+    ])
+    .unwrap();
+
+    assert_eq!(
+        loaded,
+        NativeConfig {
+            top: "base_top".to_owned(),
+            inner: NativeInnerConfig { a: 1, b: "created_b".to_owned() },
+        }
+    );
+}
+
 // Make sure that if we have a field `foo_bar` and an optional field called `foo` with a value of
 // None, we don't remove the foo_bar field from the config.
 // This test was added following bug #37984 (see bug for more details).

crates/apollo_config/src/lib.rs

@@ -64,6 +64,21 @@ pub const CONFIG_FILE_SHORT_ARG_NAME: char = 'f';
 /// The config file arg name prepended with a double dash.
 pub const CONFIG_FILE_ARG: &str = formatcp!("--{}", CONFIG_FILE_ARG_NAME);
 
+/// Arg name for selecting how the config files are interpreted.
+pub const CONFIG_FORMAT_ARG_NAME: &str = "config_format";
+
+/// Selects how the `--config_file` arguments are interpreted when loading a config.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, clap::ValueEnum)]
+pub enum ConfigFormat {
+    /// Flat dotted-key files, layered through the full pipeline (pointers, env/CLI overrides,
+    /// `#is_none` marks). This is the historical behavior that will be deprecated.
+    #[default]
+    Preset,
+    /// The first file is a nested JSON object deserialized directly with serde; any subsequent
+    /// files are flat dotted-key secret overrides applied onto it.
+    Native,
+}
+
 /// A config indicator for optional parameters.
 pub const IS_NONE_MARK: &str = "#is_none";
 /// A config indicator for a sub config.
@@ -208,6 +223,11 @@ pub enum ConfigError {
     IOError(#[from] std::io::Error),
     #[error(transparent)]
     MissingParam(#[from] serde_json::Error),
+    #[error(
+        "Native config format requires exactly two config files: the base config and the secret \
+         config."
+    )]
+    NativeModeRequiresTwoConfigFiles,
     #[error("{pointing_param} is not found.")]
     PointerSourceNotFound { pointing_param: String },
     #[error("{target_param} is not found.")]

crates/apollo_config/src/loading.rs

@@ -14,17 +14,19 @@ use command::{get_command_matches, update_config_map_by_command_args};
 use itertools::any;
 use serde::Deserialize;
 use serde_json::{json, Map, Value};
-use tracing::{error, info, instrument};
+use tracing::{debug, error, info, instrument};
 
 use crate::validators::validate_path_exists;
 use crate::{
     command,
     ConfigError,
+    ConfigFormat,
     ParamPath,
     SerializationType,
     SerializedContent,
     SerializedParam,
     CONFIG_FILE_ARG_NAME,
+    CONFIG_FORMAT_ARG_NAME,
     FIELD_SEPARATOR,
     IS_NONE_MARK,
 };
@@ -47,6 +49,68 @@ pub fn load<T: for<'a> Deserialize<'a>>(
     Ok(serde_json::from_value(nested_map)?)
 }
 
+/// Deserializes the config directly from nested JSON files (the `ConfigFormat::Native` path).
+///
+/// Requires exactly two paths. The first is the base config: a nested JSON object matching the
+/// target struct's field hierarchy. The second is a flat dotted-key secret file whose entries are
+/// written into the nested base before deserialization. Unlike the preset path, no pointers,
+/// env/CLI overrides, or `#is_none` marks are applied — the base file is expected to be a complete,
+/// resolved config.
+fn load_native<T: for<'a> Deserialize<'a>>(
+    custom_config_paths: Vec<PathBuf>,
+) -> Result<T, ConfigError> {
+    let [base_config_path, secret_config_path] =
+        custom_config_paths.as_array().ok_or(ConfigError::NativeModeRequiresTwoConfigFiles)?;
+
+    validate_path_exists(base_config_path)?;
+    info!("Loading native config file: {:?}", base_config_path);
+    let mut nested_map: Value = serde_json::from_reader(File::open(base_config_path)?)?;
+    validate_path_exists(secret_config_path)?;
+    info!("Loading native secret config file: {:?}", secret_config_path);
+
+    let secret_config: Map<String, Value> =
+        serde_json::from_reader(File::open(secret_config_path)?)?;
+    for (param_path, value) in secret_config {
+        set_nested_value(&mut nested_map, &param_path, value);
+    }
+
+    Ok(serde_json::from_value(nested_map)?)
+}
+
+/// Writes `value` into `nested_map` at the leaf named by the dotted `param_path`, descending
+/// through (and preserving the siblings of) the intermediate objects. The leaf key itself is
+/// created if absent.
+///
+/// Traversal stops without writing anything if an intermediate on the path is `null` (a `None`
+/// optional, e.g. a disabled component) or is missing/not an object. The base config is the source
+/// of truth for which subtrees exist; a secret aimed at a subtree the base omits is irrelevant, and
+/// vivifying it would produce a partial object that fails deserialization. This also avoids any
+/// panic on a type-mismatched intermediate.
+fn set_nested_value(nested_map: &mut Value, param_path: &str, value: Value) {
+    let mut segments = param_path.split(FIELD_SEPARATOR).peekable();
+    let mut entry = nested_map;
+    while let Some(segment) = segments.next() {
+        let Value::Object(map) = entry else {
+            return;
+        };
+        if segments.peek().is_none() {
+            map.insert(segment.to_owned(), value);
+            return;
+        }
+        // Descend into the intermediate; stop if it is absent or an explicit `null` (`None`).
+        match map.get_mut(segment) {
+            Some(child) if !child.is_null() => entry = child,
+            _ => {
+                debug!(
+                    "Skipping secret override {param_path:?}: intermediate {segment:?} is absent \
+                     or None in the base config."
+                );
+                return;
+            }
+        }
+    }
+}
+
 /// Deserializes a json config file, updates the values by the given arguments for the command, and
 /// set values for the pointers.
 pub fn load_and_process_config<T: for<'a> Deserialize<'a>>(
@@ -62,6 +126,16 @@ pub fn load_and_process_config<T: for<'a> Deserialize<'a>>(
     let (config_map, pointers_map) = split_pointers_map(deserialized_config_schema.clone());
     // Take param paths with corresponding descriptions, and get the matching arguments.
     let mut arg_matches = get_command_matches(&config_map, command, args)?;
+
+    let config_format =
+        arg_matches.remove_one::<ConfigFormat>(CONFIG_FORMAT_ARG_NAME).unwrap_or_default();
+    if config_format == ConfigFormat::Native {
+        let custom_config_paths: Vec<PathBuf> = arg_matches
+            .remove_many::<PathBuf>(CONFIG_FILE_ARG_NAME)
+            .map(|paths| paths.collect())
+            .unwrap_or_default();
+        return load_native(custom_config_paths);
+    }
     // Retaining values from the default config map for backward compatibility.
     let (mut values_map, types_map) = split_values_and_types(config_map);
     if ignore_default_values {