diff --git a/deployments/monitoring/README.md b/deployments/monitoring/README.md index d6ebd2b6741..fab7c64f42b 100644 --- a/deployments/monitoring/README.md +++ b/deployments/monitoring/README.md @@ -23,6 +23,22 @@ To force build (useful to enforce applying changes in docker file settings): ./deployments/monitoring/deploy_local_stack.sh up -d --build ``` +### Multiple nodes: +Set `N_NODES` (1-5, default 1) to run several sequencer nodes in real consensus, each in its own +container (`sequencer-node-0`, `sequencer-node-1`, ...). Prometheus scrapes every node, so dashboard +panels show one series per node (useful for observing proposer/validator agreement). Pass the same +`N_NODES` when tearing down. +```bash +N_NODES=3 ./deployments/monitoring/deploy_local_stack.sh up -d --build +``` + +### Real exchange-rate oracle: +By default the stack uses a dummy constant-rate oracle. Set `PRAGMA_API_KEY` to instead pull live +ETH/STRK and STRK/USD rates from Pragma (so the SNIP-35 `fee_target` tracks the real STRK/USD rate): +```bash +N_NODES=3 PRAGMA_API_KEY= ./deployments/monitoring/deploy_local_stack.sh up -d --build +``` + ## This will deploy a local stack of: - Sequencer Node Setup - Sequencer Node (using a config generated by **Sequencer Node Setup**.) diff --git a/deployments/monitoring/deploy_local_stack.sh b/deployments/monitoring/deploy_local_stack.sh index 7d9e0fd0b64..9ded57a8d48 100755 --- a/deployments/monitoring/deploy_local_stack.sh +++ b/deployments/monitoring/deploy_local_stack.sh @@ -7,11 +7,29 @@ export DOCKER_BUILDKIT export COMPOSE_DOCKER_CLI_BUILD export MONITORING_ENABLED=${MONITORING_ENABLED:-true} export FOLLOW_LOGS=${FOLLOW_LOGS:-false} -export SEQUENCER_HTTP_PORT=${SEQUENCER_HTTP_PORT:-8081} -export SEQUENCER_MONITORING_PORT=${SEQUENCER_MONITORING_PORT:-8082} -export SEQUENCER_CONFIG_PATH=${SEQUENCER_CONFIG_PATH:-/config/node_0/config.json} export SEQUENCER_ROOT_DIR=${SEQUENCER_ROOT_DIR:-$(git -C "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" rev-parse --show-toplevel)} +# Number of sequencer nodes to run in real consensus, each in its own container (max 5). Default 1. +export N_NODES=${N_NODES:-1} +if ! [[ "$N_NODES" =~ ^[1-5]$ ]]; then + echo "Error: N_NODES must be an integer between 1 and 5 (got '$N_NODES')." + exit 1 +fi +# sequencer-node-0 has no compose profile, so it always starts and stops with the project; only +# sequencer-node-1..4 are profile-gated (node1..node4; there is no node5, since N_NODES caps at 5). +# For `up`, enable a profile per extra node so only sequencer-node-0..N-1 start. For `down`, enable +# every node profile so all nodes are torn down regardless of how many were started (otherwise the +# profiled nodes survive `down`). +if [ "$1" == "down" ]; then + export COMPOSE_PROFILES="node1,node2,node3,node4" +else + profiles="" + for ((node_index = 1; node_index < N_NODES; node_index++)); do + profiles="${profiles:+$profiles,}node${node_index}" + done + export COMPOSE_PROFILES="$profiles" +fi + # Exchange-rate oracle source. When PRAGMA_API_KEY is set, point the node at the real Pragma # oracle so eth_to_strk (L1 gas price conversion) and strk_to_usd (SNIP-35 fee_target) track live # rates; otherwise fall back to the dummy constant-rate oracle (see docker-compose.yml @@ -40,7 +58,11 @@ else fi if [ "$MONITORING_ENABLED" != true ]; then - services="sequencer_node_setup dummy_recorder dummy_exchange_rate_oracle config_injector sequencer_node sequencer_simulator" + node_services="" + for ((node_index = 0; node_index < N_NODES; node_index++)); do + node_services="${node_services} sequencer-node-${node_index}" + done + services="sequencer_node_setup dummy_recorder dummy_exchange_rate_oracle config_injector${node_services} sequencer_simulator" fi echo "Running: ${docker_compose} -f ${monitoring_dir}/local/docker-compose.yml $*" diff --git a/deployments/monitoring/local/config/prometheus/prometheus.yml b/deployments/monitoring/local/config/prometheus/prometheus.yml index 87afb99c8aa..637a65c7971 100644 --- a/deployments/monitoring/local/config/prometheus/prometheus.yml +++ b/deployments/monitoring/local/config/prometheus/prometheus.yml @@ -9,4 +9,8 @@ scrape_configs: metrics_path: /monitoring/metrics static_configs: - targets: - - sequencer_node:8082 + - sequencer-node-0:8082 + - sequencer-node-1:8082 + - sequencer-node-2:8082 + - sequencer-node-3:8082 + - sequencer-node-4:8082 diff --git a/deployments/monitoring/local/docker-compose.yml b/deployments/monitoring/local/docker-compose.yml index 96274fd9772..dc69b2ed02a 100644 --- a/deployments/monitoring/local/docker-compose.yml +++ b/deployments/monitoring/local/docker-compose.yml @@ -1,3 +1,30 @@ +# Shared sequencer-node service body. Each node runs in its own container and reaches the others by +# docker DNS name (sequencer-node-0, sequencer-node-1, ...). node_1+ are gated behind profiles so +# only N_NODES of them start (see deploy_local_stack.sh). +x-sequencer-node: &sequencer_node + depends_on: + config_injector: + condition: service_completed_successfully + sequencer_node_setup: + condition: service_completed_successfully + dummy_recorder: + condition: service_started + dummy_exchange_rate_oracle: + condition: service_started + build: + context: ${SEQUENCER_ROOT_DIR} + dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/Dockerfile + args: + BUILD_MODE: debug + environment: + - RUST_LOG=${RUST_LOG} + - RUST_BACKTRACE=${RUST_BACKTRACE} + volumes: + - data:/data + - config:/config + networks: + - sequencer-network + services: prometheus: image: prom/prometheus @@ -31,7 +58,7 @@ services: - RUST_BACKTRACE=${RUST_BACKTRACE} entrypoint: "/bin/bash -c" command: > - "./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-distributed 0 --n-hybrid 0 --n-consolidated 1; + "./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-distributed 0 --n-hybrid 0 --n-consolidated ${N_NODES:-1}; cp -r ./output/data/* /data; cp -r ./output/configs/* /config" volumes: - data:/data @@ -71,66 +98,81 @@ services: build: context: ${SEQUENCER_ROOT_DIR} dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/config_injector.Dockerfile - # TODO(Tsabary): change the setup binary to output the file to a specific path, and use it in the following command. - # TODO(Tsabary): the config changes need to be more robust, probably managed through a suitable rust binary. - command: | - "cp /config/node_0/node_integration_test_config_changes.json ${SEQUENCER_CONFIG_PATH} \ - echo 'Injecting config changes...' && \ - jq '.\"recorder_url\" = \"http://dummy_recorder:8080\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_gas_price_provider_config.eth_to_strk_oracle_config.url_header_list\" = \"${ETH_STRK_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_gas_price_provider_config.eth_to_strk_oracle_config.lag_interval_seconds\" = ${ETH_STRK_ORACLE_LAG_SECONDS:-60}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_gas_price_provider_config.strk_to_usd_oracle_config.url_header_list\" = \"${STRK_USD_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_gas_price_provider_config.strk_to_usd_oracle_config.lag_interval_seconds\" = ${STRK_USD_ORACLE_LAG_SECONDS:-60}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"http_server_config.static_config.ip\" = \"0.0.0.0\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"http_server_config.static_config.port\" = ${SEQUENCER_HTTP_PORT}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"monitoring_endpoint_config.port\" = ${SEQUENCER_MONITORING_PORT}' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - # These are here to avoid using an L1 baselayer. This is because anvil was deliberately disabled in the docker test. - jq '.\"components.l1_events_scraper.execution_mode\" = \"Disabled\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_events_scraper_config.#is_none\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_events_provider_config.dummy_mode\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"components.l1_gas_price_scraper.execution_mode\" = \"Disabled\"' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - jq '.\"l1_gas_price_scraper_config.#is_none\" = true' ${SEQUENCER_CONFIG_PATH} | sponge ${SEQUENCER_CONFIG_PATH} && \ - echo 'Printing final config:' && \ - echo '----------------------------------------' && \ - cat ${SEQUENCER_CONFIG_PATH} && \ - echo '----------------------------------------' && \ - echo 'Done'" + # Inject local-stack overrides into each node's config. Single $ is interpolated by docker + # compose (host env); $$ is passed through to the container shell. For N>1 the localhost + # full-mesh P2P bootstrap addresses are rewritten to per-node DNS names (entry j -> node j, since + # the list is node-index ordered and identical across nodes). + command: + - | + set -e + N_NODES=${N_NODES:-1} + for i in $$(seq 0 $$((N_NODES - 1))); do + cfg=/config/node_$$i/config.json + cp /config/node_$$i/node_integration_test_config_changes.json "$$cfg" + jq '."recorder_url" = "http://dummy_recorder:8080"' "$$cfg" | sponge "$$cfg" + jq --arg v "${ETH_STRK_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}" '."l1_gas_price_provider_config.eth_to_strk_oracle_config.url_header_list" = $$v' "$$cfg" | sponge "$$cfg" + jq --argjson v ${ETH_STRK_ORACLE_LAG_SECONDS:-60} '."l1_gas_price_provider_config.eth_to_strk_oracle_config.lag_interval_seconds" = $$v' "$$cfg" | sponge "$$cfg" + jq --arg v "${STRK_USD_ORACLE_URL_HEADERS:-http://dummy_exchange_rate_oracle:9000/eth_to_strk_oracle}" '."l1_gas_price_provider_config.strk_to_usd_oracle_config.url_header_list" = $$v' "$$cfg" | sponge "$$cfg" + jq --argjson v ${STRK_USD_ORACLE_LAG_SECONDS:-60} '."l1_gas_price_provider_config.strk_to_usd_oracle_config.lag_interval_seconds" = $$v' "$$cfg" | sponge "$$cfg" + jq '."http_server_config.static_config.ip" = "0.0.0.0"' "$$cfg" | sponge "$$cfg" + jq '."http_server_config.static_config.port" = 8081' "$$cfg" | sponge "$$cfg" + jq '."monitoring_endpoint_config.port" = 8082' "$$cfg" | sponge "$$cfg" + jq '."components.l1_events_scraper.execution_mode" = "Disabled"' "$$cfg" | sponge "$$cfg" + jq '."l1_events_scraper_config.#is_none" = true' "$$cfg" | sponge "$$cfg" + jq '."l1_events_provider_config.dummy_mode" = true' "$$cfg" | sponge "$$cfg" + jq '."components.l1_gas_price_scraper.execution_mode" = "Disabled"' "$$cfg" | sponge "$$cfg" + jq '."l1_gas_price_scraper_config.#is_none" = true' "$$cfg" | sponge "$$cfg" + if [ "$$N_NODES" -gt 1 ]; then + for key in consensus_manager_config.network_config.bootstrap_peer_multiaddr mempool_p2p_config.network_config.bootstrap_peer_multiaddr state_sync_config.static_config.network_config.bootstrap_peer_multiaddr; do + jq --arg k "$$key" '.[$$k] |= (split(",") | to_entries | map(.key as $$j | .value | sub("/ip4/[0-9.]+/"; "/dns4/sequencer-node-\($$j)/")) | join(","))' "$$cfg" | sponge "$$cfg" + done + fi + done volumes: - config:/config networks: - sequencer-network - sequencer_node: - depends_on: - config_injector: - condition: service_completed_successfully - dummy_recorder: - condition: service_started - sequencer_node_setup: - condition: service_completed_successfully - build: - context: ${SEQUENCER_ROOT_DIR} - dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/Dockerfile - args: - BUILD_MODE: debug - environment: - - RUST_LOG=${RUST_LOG} - - RUST_BACKTRACE=${RUST_BACKTRACE} + sequencer-node-0: + <<: *sequencer_node ports: - - ${SEQUENCER_HTTP_PORT}:${SEQUENCER_HTTP_PORT} - - ${SEQUENCER_MONITORING_PORT}:${SEQUENCER_MONITORING_PORT} + - "8081:8081" + - "8082:8082" command: - "--config_file" - - "${SEQUENCER_CONFIG_PATH}" - volumes: - - data:/data - - config:/config - networks: - - sequencer-network + - "/config/node_0/config.json" + + sequencer-node-1: + <<: *sequencer_node + profiles: ["node1"] + command: + - "--config_file" + - "/config/node_1/config.json" + + sequencer-node-2: + <<: *sequencer_node + profiles: ["node2"] + command: + - "--config_file" + - "/config/node_2/config.json" + + sequencer-node-3: + <<: *sequencer_node + profiles: ["node3"] + command: + - "--config_file" + - "/config/node_3/config.json" + + sequencer-node-4: + <<: *sequencer_node + profiles: ["node4"] + command: + - "--config_file" + - "/config/node_4/config.json" sequencer_simulator: depends_on: - - sequencer_node + - sequencer-node-0 build: context: ${SEQUENCER_ROOT_DIR} dockerfile: ${SEQUENCER_ROOT_DIR}/deployments/images/sequencer/simulator.Dockerfile @@ -141,10 +183,10 @@ services: entrypoint: "/bin/bash -c" command: > "./target/debug/sequencer_simulator \ - --http-url http://sequencer_node \ - --http-port ${SEQUENCER_HTTP_PORT} \ - --monitoring-url http://sequencer_node \ - --monitoring-port ${SEQUENCER_MONITORING_PORT} \ + --http-url http://sequencer-node-0 \ + --http-port 8081 \ + --monitoring-url http://sequencer-node-0 \ + --monitoring-port 8082 \ $$(if [ \"$$SIMULATOR_RUN_FOREVER\" = \"true\" ]; then echo '--run-forever'; fi)" networks: - sequencer-network