feat: update subnet configuration and enhance process management

- Changed subnet ID and updated parent registry and gateway addresses in `ipc-subnet-config.yml` for improved network configuration.
- Enhanced `exec.sh` to check process status more reliably by capturing output and trimming whitespace.
- Improved `health.sh` to ensure proper cleanup of stale RocksDB locks and added fallback mechanisms for stopping processes, enhancing robustness during node shutdown.
- Added validation for extracted subnet ID against on-chain subnet list to ensure consistency and correctness in subnet management.

Author: phutchins

scripts/ipc-subnet-manager/ipc-subnet-config.yml

@@ -4,17 +4,17 @@
 # Subnet Configuration
 subnet:
   # Subnet ID - get this from your subnet creation
-  id: "/r314159/t410fdg2vwrukcxeopg3sfslrmsf2fkxjyzfldcnf7kq"
+  id: "/r314159/t410fko243732vbebcpg32q6lbcnhmmqlkfyvu36ibdi"
   # Parent chain RPC endpoint
   parent_rpc: "https://api.calibration.node.glif.io/rpc/v1"
   #parent_rpc: "http://localhost:8555"
 
   # Parent chain ID
   parent_chain_id: "/r314159"
   # Parent registry contract address
-  parent_registry: "0x3527b860bfe033601b5cfda1dbaaaeb0e81033be"
+  parent_registry: "0x618dc98241aa36710d83de2927eeb50ccd1c853d"
   # Parent gateway contract address
-  parent_gateway: "0x46eb516b535b990825a3171a28548af072975296"
+  parent_gateway: "0x9e78a3c81023b8f4edb16f2975893c76b3265a82"
 # Validator Nodes (GCP hosts: ipc-chain-test-1, ipc-chain-test-2, ipc-chain-test-3)
 # ip: external IP (for SSH, RPC). internal_ip: for CometBFT/libp2p peer connections (same VPC)
 validators:
@@ -129,8 +129,8 @@ ipc_cli:
     network_type: "fevm"
     provider_http: "https://api.calibration.node.glif.io/rpc/v1"
     #provider_http: "http://localhost:8555"
-    registry_addr: "0x3527b860bfe033601b5cfda1dbaaaeb0e81033be"
-    gateway_addr: "0x46eb516b535b990825a3171a28548af072975296"
+    registry_addr: "0x618dc98241aa36710d83de2927eeb50ccd1c853d"
+    gateway_addr: "0x9e78a3c81023b8f4edb16f2975893c76b3265a82"
   # Child subnet configuration (this subnet)
   child:
     # Uses subnet.id from above

scripts/ipc-subnet-manager/lib/exec.sh

@@ -155,7 +155,9 @@ check_process_running() {
         local ip=$(get_config_value "validators[$validator_idx].ip")
         local ssh_user=$(get_config_value "validators[$validator_idx].ssh_user")
         local ipc_user=$(get_config_value "validators[$validator_idx].ipc_user")
-        ssh_check_process "$ip" "$ssh_user" "$ipc_user" "$process_pattern"
+        local status
+        status=$(ssh_check_process "$ip" "$ssh_user" "$ipc_user" "$process_pattern" 2>/dev/null | tr -d '\r\n ')
+        [ "$status" = "running" ]
     fi
 }
 

scripts/ipc-subnet-manager/lib/health.sh

@@ -34,11 +34,18 @@ stop_all_nodes() {
         log_info "Stopping $name..."
         kill_process "$idx" "ipc-cli node start"
         # Also stop child processes that can keep RocksDB LOCK files open.
+        kill_process "$idx" "fendermint"
         kill_process "$idx" "/target/release/fendermint"
         kill_process "$idx" "fendermint/app/src/service/node.rs"
         kill_process "$idx" "cometbft start"
+        kill_process "$idx" "cometbft"
         kill_process "$idx" "/cometbft"
 
+        # Fallback non-sudo process kill when sudo -n is unavailable.
+        exec_on_host_simple "$idx" "pkill -f \"ipc-cli node start\" >/dev/null 2>&1 || true" || true
+        exec_on_host_simple "$idx" "pkill -f \"fendermint\" >/dev/null 2>&1 || true" || true
+        exec_on_host_simple "$idx" "pkill -f \"cometbft\" >/dev/null 2>&1 || true" || true
+
         # Best-effort systemd stop (some hosts run ipc-node service directly).
         if ! is_local_mode; then
             local ip
@@ -51,9 +58,30 @@ stop_all_nodes() {
 
         # Wait a moment for graceful shutdown
         sleep 2
+
+        # Stale LOCK can remain when prior process exits uncleanly.
+        cleanup_stale_rocksdb_lock "$idx"
     done
 }
 
+cleanup_stale_rocksdb_lock() {
+    local validator_idx="$1"
+    local name="${VALIDATORS[$validator_idx]}"
+    local node_home
+    node_home=$(get_node_home "$validator_idx")
+    local lock_path="$node_home/fendermint/data/rocksdb/LOCK"
+
+    if exec_on_host_simple "$validator_idx" "pgrep -f \"fendermint|cometbft|ipc-cli node start\" >/dev/null 2>&1"; then
+        return 0
+    fi
+
+    if exec_on_host_simple "$validator_idx" "[ -f $lock_path ]"; then
+        if exec_on_host_simple "$validator_idx" "rm -f $lock_path"; then
+            log_warn "Removed stale RocksDB lock on $name: $lock_path"
+        fi
+    fi
+}
+
 stop_all_storage_nodes() {
     for idx in "${!VALIDATORS[@]}"; do
         local name="${VALIDATORS[$idx]}"
@@ -90,6 +118,7 @@ start_validator_node() {
     local subnet_id=$(get_config_value "subnet.id")
 
     log_info "Starting $name..."
+    cleanup_stale_rocksdb_lock "$validator_idx"
 
     # Use wrapper script to set env vars reliably (avoids SSH quoting issues with sudo su -c '...').
     # resolver_enabled() requires: !listen_addr.is_empty() && subnet_id != UNDEF
@@ -666,6 +695,23 @@ EOF
         }
     ' "$ipc_config_file")
 
+    # Validate extracted subnet ID against on-chain subnet list.
+    # If TOML extraction picked a stale local entry, prefer the latest on-chain child subnet.
+    local onchain_subnets
+    onchain_subnets=$($ipc_binary_expanded subnet list --parent "$parent_chain_id" 2>/dev/null | awk -v parent="$parent_chain_id" '
+        $1 ~ ("^" parent "/t[a-z0-9]+$") { print $1 }
+    ')
+    if [ -n "$onchain_subnets" ]; then
+        if [ -z "$subnet_id" ] || ! echo "$onchain_subnets" | grep -Fxq "$subnet_id"; then
+            local onchain_latest
+            onchain_latest=$(echo "$onchain_subnets" | tail -1)
+            if [ -n "$onchain_latest" ]; then
+                log_warn "Extracted subnet ID was not on-chain; using latest on-chain subnet: $onchain_latest" >&2
+                subnet_id="$onchain_latest"
+            fi
+        fi
+    fi
+
     if [ -z "$subnet_id" ]; then
         log_error "Could not extract subnet ID from IPC config at $ipc_config_file" >&2
         log_info "Full CLI output:" >&2