deployment: add N_NODES flag to run multiple nodes in local monitoring stack

deployments/monitoring/README.md

@@ -23,6 +23,22 @@ To force build (useful to enforce applying changes in docker file settings):
 ./deployments/monitoring/deploy_local_stack.sh up -d --build
 ```
 
+### Multiple nodes:
+Set `N_NODES` (1-5, default 1) to run several sequencer nodes in real consensus, each in its own
+container (`sequencer-node-0`, `sequencer-node-1`, ...). Prometheus scrapes every node, so dashboard
+panels show one series per node (useful for observing proposer/validator agreement). Pass the same
+`N_NODES` when tearing down.
+```bash
+N_NODES=3 ./deployments/monitoring/deploy_local_stack.sh up -d --build
+```
+
+### Real exchange-rate oracle:
+By default the stack uses a dummy constant-rate oracle. Set `PRAGMA_API_KEY` to instead pull live
+ETH/STRK and STRK/USD rates from Pragma (so the SNIP-35 `fee_target` tracks the real STRK/USD rate):
+```bash
+N_NODES=3 PRAGMA_API_KEY=<key> ./deployments/monitoring/deploy_local_stack.sh up -d --build
+```
+
 ## This will deploy a local stack of:
 - Sequencer Node Setup
 - Sequencer Node (using a config generated by **Sequencer Node Setup**.)

deployments/monitoring/deploy_local_stack.sh

@@ -7,11 +7,29 @@ export DOCKER_BUILDKIT
 export COMPOSE_DOCKER_CLI_BUILD
 export MONITORING_ENABLED=${MONITORING_ENABLED:-true}
 export FOLLOW_LOGS=${FOLLOW_LOGS:-false}
-export SEQUENCER_HTTP_PORT=${SEQUENCER_HTTP_PORT:-8081}
-export SEQUENCER_MONITORING_PORT=${SEQUENCER_MONITORING_PORT:-8082}
-export SEQUENCER_CONFIG_PATH=${SEQUENCER_CONFIG_PATH:-/config/node_0/config.json}
 export SEQUENCER_ROOT_DIR=${SEQUENCER_ROOT_DIR:-$(git -C "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" rev-parse --show-toplevel)}
 
+# Number of sequencer nodes to run in real consensus, each in its own container (max 5). Default 1.
+export N_NODES=${N_NODES:-1}
+if ! [[ "$N_NODES" =~ ^[1-5]$ ]]; then
+  echo "Error: N_NODES must be an integer between 1 and 5 (got '$N_NODES')."
+  exit 1
+fi
+# sequencer-node-0 has no compose profile, so it always starts and stops with the project; only
+# sequencer-node-1..4 are profile-gated (node1..node4; there is no node5, since N_NODES caps at 5).
+# For `up`, enable a profile per extra node so only sequencer-node-0..N-1 start. For `down`, enable
+# every node profile so all nodes are torn down regardless of how many were started (otherwise the
+# profiled nodes survive `down`).
+if [ "$1" == "down" ]; then
+  export COMPOSE_PROFILES="node1,node2,node3,node4"
+else
+  profiles=""
+  for ((node_index = 1; node_index < N_NODES; node_index++)); do
+    profiles="${profiles:+$profiles,}node${node_index}"
+  done
+  export COMPOSE_PROFILES="$profiles"
+fi
+
 # Exchange-rate oracle source. When PRAGMA_API_KEY is set, point the node at the real Pragma
 # oracle so eth_to_strk (L1 gas price conversion) and strk_to_usd (SNIP-35 fee_target) track live
 # rates; otherwise fall back to the dummy constant-rate oracle (see docker-compose.yml
@@ -40,7 +58,11 @@ else
 fi
 
 if [ "$MONITORING_ENABLED" != true ]; then
-  services="sequencer_node_setup dummy_recorder dummy_exchange_rate_oracle config_injector sequencer_node sequencer_simulator"
+  node_services=""
+  for ((node_index = 0; node_index < N_NODES; node_index++)); do
+    node_services="${node_services} sequencer-node-${node_index}"
+  done
+  services="sequencer_node_setup dummy_recorder dummy_exchange_rate_oracle config_injector${node_services} sequencer_simulator"
 fi
 
 echo "Running: ${docker_compose} -f ${monitoring_dir}/local/docker-compose.yml $*"

deployments/monitoring/local/config/prometheus/prometheus.yml

@@ -9,4 +9,8 @@ scrape_configs:
    metrics_path: /monitoring/metrics
    static_configs:
     - targets:
-       - sequencer_node:8082
+       - sequencer-node-0:8082
+       - sequencer-node-1:8082
+       - sequencer-node-2:8082
+       - sequencer-node-3:8082
+       - sequencer-node-4:8082

deployments/monitoring/local/docker-compose.yml

@@ -1,3 +1,30 @@
+# Shared sequencer-node service body. Each node runs in its own container and reaches the others by
+# docker DNS name (sequencer-node-0, sequencer-node-1, ...). node_1+ are gated behind profiles so
+# only N_NODES of them start (see deploy_local_stack.sh).
+x-sequencer-node: &sequencer_node
+  depends_on:
+    config_injector:
+      condition: service_completed_successfully
+    sequencer_node_setup:
+      condition: service_completed_successfully
+    dummy_recorder:
+      condition: service_started
+    dummy_exchange_rate_oracle:
+      condition: service_started
+  build:
+    context: ${SEQUENCER_ROOT_DIR}
+    dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/Dockerfile
+    args:
+      BUILD_MODE: debug
+  environment:
+    - RUST_LOG=${RUST_LOG}
+    - RUST_BACKTRACE=${RUST_BACKTRACE}
+  volumes:
+    - data:/data
+    - config:/config
+  networks:
+    - sequencer-network
+
 services:
   prometheus:
     image: prom/prometheus
@@ -31,7 +58,7 @@ services:
       - RUST_BACKTRACE=${RUST_BACKTRACE}
     entrypoint: "/bin/bash -c"
     command: >
-      "./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-distributed 0 --n-hybrid 0 --n-consolidated 1;
+      "./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-distributed 0 --n-hybrid 0 --n-consolidated ${N_NODES:-1};
       cp -r ./output/data/* /data; cp -r ./output/configs/* /config"
     volumes:
       - data:/data
@@ -71,66 +98,81 @@ services:
     build:
       context: ${SEQUENCER_ROOT_DIR}
       dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/config_injector.Dockerfile
-    # TODO(Tsabary): change the setup binary to output the file to a specific path, and use it in the following command.
-    # TODO(Tsabary): the config changes need to be more robust, probably managed through a suitable rust binary.
-    command: |
-      "cp /config/node_0/node_integration_test_config_changes.json ${SEQUENCER_CONFIG_PATH} \
-      echo 'Injecting config changes...' && \
-      jq '.\"recorder_url\" = \"http://dummy_recorder:8080\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_gas_price_provider_config.eth_to_strk_oracle_config.url_header_list\" = \"${ETH_STRK_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_gas_price_provider_config.eth_to_strk_oracle_config.lag_interval_seconds\" = ${ETH_STRK_ORACLE_LAG_SECONDS:-60}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_gas_price_provider_config.strk_to_usd_oracle_config.url_header_list\" = \"${STRK_USD_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_gas_price_provider_config.strk_to_usd_oracle_config.lag_interval_seconds\" = ${STRK_USD_ORACLE_LAG_SECONDS:-60}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"http_server_config.static_config.ip\" = \"0.0.0.0\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"http_server_config.static_config.port\" = ${SEQUENCER_HTTP_PORT}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"monitoring_endpoint_config.port\" = ${SEQUENCER_MONITORING_PORT}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      # These are here to avoid using an L1 baselayer. This is because anvil was deliberately disabled in the docker test. 
-      jq '.\"components.l1_events_scraper.execution_mode\" = \"Disabled\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_events_scraper_config.#is_none\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_events_provider_config.dummy_mode\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"components.l1_gas_price_scraper.execution_mode\" = \"Disabled\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      jq '.\"l1_gas_price_scraper_config.#is_none\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \
-      echo 'Printing final config:' && \
-      echo '----------------------------------------' && \
-      cat ${SEQUENCER_CONFIG_PATH} && \
-      echo '----------------------------------------' && \
-      echo 'Done'"
+    # Inject local-stack overrides into each node's config. Single $ is interpolated by docker
+    # compose (host env); $$ is passed through to the container shell. For N>1 the localhost
+    # full-mesh P2P bootstrap addresses are rewritten to per-node DNS names (entry j -> node j, since
+    # the list is node-index ordered and identical across nodes).
+    command:
+      - |
+        set -e
+        N_NODES=${N_NODES:-1}
+        for i in $$(seq 0 $$((N_NODES - 1))); do
+          cfg=/config/node_$$i/config.json
+          cp /config/node_$$i/node_integration_test_config_changes.json "$$cfg"
+          jq '."recorder_url" = "http://dummy_recorder:8080"' "$$cfg" | sponge "$$cfg"
+          jq --arg v "${ETH_STRK_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}" '."l1_gas_price_provider_config.eth_to_strk_oracle_config.url_header_list" = $$v' "$$cfg" | sponge "$$cfg"
+          jq --argjson v ${ETH_STRK_ORACLE_LAG_SECONDS:-60} '."l1_gas_price_provider_config.eth_to_strk_oracle_config.lag_interval_seconds" = $$v' "$$cfg" | sponge "$$cfg"
+          jq --arg v "${STRK_USD_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}" '."l1_gas_price_provider_config.strk_to_usd_oracle_config.url_header_list" = $$v' "$$cfg" | sponge "$$cfg"
+          jq --argjson v ${STRK_USD_ORACLE_LAG_SECONDS:-60} '."l1_gas_price_provider_config.strk_to_usd_oracle_config.lag_interval_seconds" = $$v' "$$cfg" | sponge "$$cfg"
+          jq '."http_server_config.static_config.ip" = "0.0.0.0"' "$$cfg" | sponge "$$cfg"
+          jq '."http_server_config.static_config.port" = 8081' "$$cfg" | sponge "$$cfg"
+          jq '."monitoring_endpoint_config.port" = 8082' "$$cfg" | sponge "$$cfg"
+          jq '."components.l1_events_scraper.execution_mode" = "Disabled"' "$$cfg" | sponge "$$cfg"
+          jq '."l1_events_scraper_config.#is_none" = true' "$$cfg" | sponge "$$cfg"
+          jq '."l1_events_provider_config.dummy_mode" = true' "$$cfg" | sponge "$$cfg"
+          jq '."components.l1_gas_price_scraper.execution_mode" = "Disabled"' "$$cfg" | sponge "$$cfg"
+          jq '."l1_gas_price_scraper_config.#is_none" = true' "$$cfg" | sponge "$$cfg"
+          if [ "$$N_NODES" -gt 1 ]; then
+            for key in consensus_manager_config.network_config.bootstrap_peer_multiaddr mempool_p2p_config.network_config.bootstrap_peer_multiaddr state_sync_config.static_config.network_config.bootstrap_peer_multiaddr; do
+              jq --arg k "$$key" '.[$$k] |= (split(",") | to_entries | map(.key as $$j | .value | sub("/ip4/[0-9.]+/"; "/dns4/sequencer-node-\($$j)/")) | join(","))' "$$cfg" | sponge "$$cfg"
+            done
+          fi
+        done
     volumes:
       - config:/config
     networks:
       - sequencer-network
 
-  sequencer_node:
-    depends_on:
-      config_injector:
-        condition: service_completed_successfully
-      dummy_recorder:
-        condition: service_started
-      sequencer_node_setup:
-        condition: service_completed_successfully
-    build:
-      context: ${SEQUENCER_ROOT_DIR}
-      dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/Dockerfile
-      args:
-        BUILD_MODE: debug
-    environment:
-      - RUST_LOG=${RUST_LOG}
-      - RUST_BACKTRACE=${RUST_BACKTRACE}
+  sequencer-node-0:
+    <<: *sequencer_node
     ports:
-      - ${SEQUENCER_HTTP_PORT}:${SEQUENCER_HTTP_PORT}
-      - ${SEQUENCER_MONITORING_PORT}:${SEQUENCER_MONITORING_PORT}
+      - "8081:8081"
+      - "8082:8082"
     command:
       - "--config_file"
-      - "${SEQUENCER_CONFIG_PATH}"
-    volumes:
-      - data:/data
-      - config:/config
-    networks:
-      - sequencer-network
+      - "/config/node_0/config.json"
+
+  sequencer-node-1:
+    <<: *sequencer_node
+    profiles: ["node1"]
+    command:
+      - "--config_file"
+      - "/config/node_1/config.json"
+
+  sequencer-node-2:
+    <<: *sequencer_node
+    profiles: ["node2"]
+    command:
+      - "--config_file"
+      - "/config/node_2/config.json"
+
+  sequencer-node-3:
+    <<: *sequencer_node
+    profiles: ["node3"]
+    command:
+      - "--config_file"
+      - "/config/node_3/config.json"
+
+  sequencer-node-4:
+    <<: *sequencer_node
+    profiles: ["node4"]
+    command:
+      - "--config_file"
+      - "/config/node_4/config.json"
 
   sequencer_simulator:
     depends_on:
-      - sequencer_node
+      - sequencer-node-0
     build:
       context: ${SEQUENCER_ROOT_DIR}
       dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/simulator.Dockerfile
@@ -141,10 +183,10 @@ services:
     entrypoint: "/bin/bash -c"
     command: >
       "./target/debug/sequencer_simulator \
-      --http-url http://sequencer_node \
-      --http-port ${SEQUENCER_HTTP_PORT} \
-      --monitoring-url http://sequencer_node \
-      --monitoring-port ${SEQUENCER_MONITORING_PORT} \
+      --http-url http://sequencer-node-0 \
+      --http-port 8081 \
+      --monitoring-url http://sequencer-node-0 \
+      --monitoring-port 8082 \
       $$(if [ \"$$SIMULATOR_RUN_FOREVER\" = \"true\" ]; then echo '--run-forever'; fi)"
     networks:
       - sequencer-network