apollo_deployments: distributed layout jsonnet test

Author: nimrod-starkware

crates/apollo_deployments/jsonnet/lib/layouts/distributed.libsonnet

@@ -0,0 +1,97 @@
+// Distributed layout structure.
+local catalog = import '../infra/catalog.libsonnet';
+local templates = import '../infra/templates.libsonnet';
+{
+  reactive:: catalog.reactive,
+  active:: catalog.active,
+  // Hosted-but-not-remote-served reactive components (LocalExecutionWithRemoteDisabled when hosted).
+  localOnly:: ['config_manager', 'mempool_p2p'],
+
+  // Canonical per-component infra ports (shared across layouts).
+  ports:: templates.componentPorts,
+  serviceDns:: {
+    batcher: 'sequencer-batcher-service',
+    class_manager: 'sequencer-class-manager-service',
+    committer: 'sequencer-committer-service',
+    gateway: 'sequencer-gateway-service',
+    l1: 'sequencer-l1-service',
+    mempool: 'sequencer-mempool-service',
+    proof_manager: 'sequencer-proof-manager-service',
+    sierra_compiler: 'sequencer-sierra-compiler-service',
+    signature_manager: 'sequencer-signature-manager-service',
+    state_sync: 'sequencer-state-sync-service',
+  },
+  // Remote-client tuning of components homed in each service (get_scale_policy + get_retries).
+  // Gateway/SierraCompiler are AutoScaled (idle 0); L1 uses RETRIES_FOR_L1_SERVICES (0); rest static.
+  scale:: {
+    batcher: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    class_manager: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    committer: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    gateway: { idle: templates.IDLE_AUTO_SCALED, retries: templates.RETRIES_DEFAULT },
+    l1: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_L1 },
+    mempool: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    proof_manager: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    sierra_compiler: { idle: templates.IDLE_AUTO_SCALED, retries: templates.RETRIES_DEFAULT },
+    signature_manager: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+    state_sync: { idle: templates.IDLE_STATIC, retries: templates.RETRIES_DEFAULT },
+  },
+  // Per service: components hosted (runs) and reactive components consumed remotely (uses).
+  // Everything not listed is Disabled. Transcribed from distributed.rs get_*_component_config.
+  services:: {
+    batcher: {
+      runs: ['batcher', 'config_manager', 'monitoring_endpoint'],
+      uses: ['class_manager', 'committer', 'l1_events_provider', 'mempool', 'proof_manager'],
+    },
+    class_manager: {
+      runs: ['class_manager', 'config_manager', 'monitoring_endpoint'],
+      uses: ['sierra_compiler'],
+    },
+    committer: {
+      runs: ['committer', 'config_manager', 'monitoring_endpoint'],
+      uses: ['batcher'],
+    },
+    consensus_manager: {
+      runs: ['config_manager', 'consensus_manager', 'monitoring_endpoint'],
+      uses: ['batcher', 'class_manager', 'l1_gas_price_provider', 'proof_manager', 'state_sync', 'signature_manager'],
+    },
+    http_server: {
+      runs: ['config_manager', 'http_server', 'monitoring_endpoint'],
+      uses: ['gateway'],
+    },
+    gateway: {
+      runs: ['gateway', 'config_manager', 'monitoring_endpoint'],
+      uses: ['class_manager', 'mempool', 'proof_manager', 'state_sync'],
+    },
+    l1: {
+      runs: [
+        'l1_gas_price_provider',
+        'l1_events_provider',
+        'config_manager',
+        'l1_gas_price_scraper',
+        'l1_events_scraper',
+        'monitoring_endpoint',
+      ],
+      uses: ['state_sync', 'batcher'],
+    },
+    proof_manager: {
+      runs: ['proof_manager', 'config_manager', 'monitoring_endpoint'],
+      uses: [],
+    },
+    mempool: {
+      runs: ['mempool', 'mempool_p2p', 'config_manager', 'monitoring_endpoint'],
+      uses: ['class_manager', 'gateway', 'proof_manager'],
+    },
+    sierra_compiler: {
+      runs: ['sierra_compiler', 'config_manager', 'monitoring_endpoint'],
+      uses: [],
+    },
+    signature_manager: {
+      runs: ['signature_manager', 'config_manager', 'monitoring_endpoint'],
+      uses: [],
+    },
+    state_sync: {
+      runs: ['state_sync', 'config_manager', 'monitoring_endpoint'],
+      uses: ['class_manager'],
+    },
+  },
+}

crates/apollo_deployments/jsonnet/services/distributed.jsonnet

@@ -0,0 +1,8 @@
+// Generates the `components` map for every service of the distributed layout.
+local derive = import '../lib/infra/derive.libsonnet';
+local distributed = import '../lib/layouts/distributed.libsonnet';
+
+{
+  [service]: derive.infraFor(distributed, service)
+  for service in std.objectFields(distributed.services)
+}

crates/apollo_deployments/src/deployment_definitions_test.rs

@@ -10,7 +10,11 @@ use strum::IntoEnumIterator;
 use tempfile::NamedTempFile;
 
 use crate::deployment_definitions::ComponentConfigInService;
-use crate::jsonnet::{test_consolidated_infra_matches_rust, test_hybrid_infra_matches_rust};
+use crate::jsonnet::{
+    test_consolidated_infra_matches_rust,
+    test_distributed_infra_matches_rust,
+    test_hybrid_infra_matches_rust,
+};
 use crate::service::NodeType;
 use crate::test_utils::SecretsConfigOverride;
 
@@ -36,6 +40,16 @@ fn consolidated_infra_matches_rust() {
     test_consolidated_infra_matches_rust();
 }
 
+/// Verifies the jsonnet distributed infra config matches the Rust deployment definitions
+/// (distributed.rs).
+#[test]
+fn distributed_infra_matches_rust() {
+    env::set_current_dir(resolve_project_relative_path("").unwrap())
+        .expect("Couldn't set working dir.");
+
+    test_distributed_infra_matches_rust();
+}
+
 /// Test that the deployment file is up to date.
 #[test]
 fn deployment_files_are_up_to_date() {

crates/apollo_deployments/src/jsonnet.rs

@@ -6,6 +6,7 @@ use serde_json::Value;
 use strum::IntoEnumIterator;
 
 use crate::deployments::consolidated::ConsolidatedNodeServiceName;
+use crate::deployments::distributed::DistributedNodeServiceName;
 use crate::deployments::hybrid::HybridNodeServiceName;
 use crate::service::{GetComponentConfigs, NodeService, NodeType};
 
@@ -69,6 +70,11 @@ pub fn test_consolidated_infra_matches_rust() {
     assert_infra_matches_rust::<ConsolidatedNodeServiceName>();
 }
 
+/// The jsonnet distributed infra matches `deployments/distributed.rs`.
+pub fn test_distributed_infra_matches_rust() {
+    assert_infra_matches_rust::<DistributedNodeServiceName>();
+}
+
 /// Clones a `components` map with `url` and `port` removed from each component object — the two
 /// fields the Rust config leaves as deploy-time placeholders, so they can't be compared against the
 /// jsonnet's baked-in real values.