apollo_integration_tests,apollo_node_config: load integration-test node config via native format

Make the integration-test harness emit a nested native config artifact plus a
secrets file and boot the node with --config_format native (two --config_file
paths: nested base first, flat secrets second), instead of the legacy preset
single-file path. Adds as_native_value/dump_native_config_file alongside the
retained preset as_value, a {} secrets file, and node_config_paths() returning
[base, secrets]; both spawn_run_node call sites pass the two-file vector.

This is the prerequisite that lets the integration suite exercise the native
load path before the preset path is torn down.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: nimrod-starkware

crates/apollo_integration_tests/src/executable_setup.rs

@@ -11,6 +11,7 @@ use tempfile::{tempdir, TempDir};
 use tokio::fs::create_dir_all;
 
 const NODE_CONFIG_CHANGES_FILE_PATH: &str = "node_integration_test_config_changes.json";
+const NODE_SECRETS_FILE_PATH: &str = "node_integration_test_secrets.json";
 
 #[derive(Debug, Clone)]
 pub struct NodeExecutableId {
@@ -48,8 +49,10 @@ pub struct ExecutableSetup {
     pub node_executable_id: NodeExecutableId,
     // Client for checking liveness of the sequencer node.
     pub monitoring_client: MonitoringClient,
-    // Path to the node configuration file.
+    // Path to the nested native base config file (consumed first by the native loader).
     pub node_config_path: PathBuf,
+    // Path to the (empty) secrets file overlaid onto the base by the native loader.
+    pub node_secrets_path: PathBuf,
     // Config.
     pub base_app_config: DeploymentBaseAppConfig,
     // Handles for the config files, maintained so the files are not deleted. Since
@@ -84,17 +87,29 @@ impl ExecutableSetup {
         let monitoring_client = MonitoringClient::new(SocketAddr::new(*ip, *port));
 
         let config_path = node_config_dir.join(NODE_CONFIG_CHANGES_FILE_PATH);
-        base_app_config.dump_config_file(&config_path);
+        base_app_config.dump_native_config_file(&config_path);
+
+        // The native loader requires a secrets file overlaid onto the base. The harness carries no
+        // secrets, so emit an empty JSON object.
+        let secrets_path = node_config_dir.join(NODE_SECRETS_FILE_PATH);
+        std::fs::write(&secrets_path, "{}").expect("Should be able to write secrets file");
 
         Self {
             node_executable_id,
             monitoring_client,
             base_app_config,
             node_config_dir_handle,
             node_config_path: config_path,
+            node_secrets_path: secrets_path,
         }
     }
 
+    /// Config files passed to the native loader, base first then secrets, matching the
+    /// `[base, secret]` arity expected by `load_native`.
+    pub(crate) fn node_config_paths(&self) -> Vec<PathBuf> {
+        vec![self.node_config_path.clone(), self.node_secrets_path.clone()]
+    }
+
     pub fn modify_config<F>(&mut self, modify_config_fn: F)
     where
         F: Fn(&mut SequencerNodeConfig),
@@ -115,8 +130,8 @@ impl ExecutableSetup {
         self.base_app_config.get_config()
     }
 
-    /// Creates a config file for the sequencer node for an integration test.
+    /// Re-emits the native base config file for the sequencer node after a config change.
     pub fn dump_config_file_changes(&self) {
-        self.base_app_config.dump_config_file(&self.node_config_path);
+        self.base_app_config.dump_native_config_file(&self.node_config_path);
     }
 }

crates/apollo_integration_tests/src/integration_test_manager.rs

@@ -281,7 +281,7 @@ impl NodeSetup {
     pub fn run_service(&self, service: NodeService) -> AbortOnDropHandle<()> {
         let executable_setup = self.get_executable_by_service(service);
         spawn_run_node(
-            vec![executable_setup.node_config_path.clone()],
+            executable_setup.node_config_paths(),
             executable_setup.node_executable_id.clone().into(),
         )
     }
@@ -294,7 +294,7 @@ impl NodeSetup {
                 (
                     *service,
                     spawn_run_node(
-                        vec![executable.node_config_path.clone()],
+                        executable.node_config_paths(),
                         executable.node_executable_id.clone().into(),
                     ),
                 )

crates/apollo_node/src/test_utils/node_runner.rs

@@ -91,9 +91,9 @@ async fn spawn_node_child_process(
     info!("Getting the node executable.");
     let node_executable = get_node_executable_path();
 
-    // Interpret the config files with the legacy flat-preset loader (the node's default), kept
-    // explicit so this stays a clear switch point if/when tests move to "native" loading.
-    let config_file_args: Vec<String> = ["--config_format".to_string(), "preset".to_string()]
+    // Interpret the config files with the native loader: the paths are a nested base config
+    // followed by a flat secrets file, matching `load_native`'s `[base, secret]` arity.
+    let config_file_args: Vec<String> = ["--config_format".to_string(), "native".to_string()]
         .into_iter()
         .chain(node_config_paths.into_iter().flat_map(|path| {
             let path_str = path.to_str().expect("Invalid path").to_string();

crates/apollo_node_config/src/config_utils.rs

@@ -196,6 +196,13 @@ impl DeploymentBaseAppConfig {
         preset
     }
 
+    /// Returns the nested config as JSON, matching the `SequencerNodeConfig` field hierarchy.
+    /// This is the artifact consumed by the `ConfigFormat::Native` loader (as the base config),
+    /// in contrast to the flat preset produced by `as_value`.
+    pub fn as_native_value(&self) -> Value {
+        serde_json::to_value(&self.config).expect("Should be able to serialize config to value")
+    }
+
     // TODO(Tsabary): unify path types throughout.
     pub fn dump_config_file(&self, config_path: &Path) {
         let value = self.as_value();
@@ -204,6 +211,15 @@ impl DeploymentBaseAppConfig {
             config_path.to_str().expect("Should be able to convert path to string"),
         );
     }
+
+    /// Dumps the nested native base config (see `as_native_value`) to `config_path`.
+    pub fn dump_native_config_file(&self, config_path: &Path) {
+        let value = self.as_native_value();
+        serialize_to_file(
+            &value,
+            config_path.to_str().expect("Should be able to convert path to string"),
+        );
+    }
 }
 
 pub fn load_and_validate_config(