[SharovBot] engineapi: fix flaky tests caused by TOCTOU port races (#20328)

## Problem

Tests in `execution/engineapi` were failing intermittently in the
`race-tests / tests-linux (ubuntu-latest, execution-other)` CI job:

- `TestEngineApiHighGasContractsFillBlock` → `listen tcp
127.0.0.1:46053: bind: address already in use`
- `TestEngineApiBlockGasOverflowSpillsToNextBlock` → `empty response
from JSON-RPC server`

Observed on commits a9db702 and acc9898 (2 out of 10 recent
commits).

**Root cause:** The test infrastructure used `freeport.NextFreePort()`
to pick a free port, then later called `net.Listen()` with that port.
This TOCTOU (time-of-check/time-of-use) race meant another concurrent
test could bind the same port in between, causing `address already in
use`. Separately, the server might not be ready when the first client
request arrived, causing `empty response from JSON-RPC server`.

## Fix

Pre-create `net.Listener` instances on port `:0` (kernel-assigned,
guaranteed free and already bound) before any server starts, then hand
the bound socket directly to `StartHTTPEndpoint`. The kernel guarantees
no other process can steal the port once it's bound.

**Changes:**
- `node/endpoints.go`: add optional `Listener` field to
`HttpEndpointConfig`; if set, skip the `net.Listen` call and use the
pre-bound socket
- `cmd/rpcdaemon/cli/httpcfg/http_cfg.go`: add `HttpListener` /
`AuthRpcListener` fields to `HttpCfg`
- `cmd/rpcdaemon/cli/config.go`: wire up pre-created listeners in both
the regular RPC and engine API listener paths
- `execution/engineapi/engineapitester/engine_api_tester.go`: replace
`freeport.NextFreePort()` with `net.Listen("tcp", "127.0.0.1:0")` for
the JSON-RPC and engine API ports

## Testing

- `go build ./...` ✅
- No `*_test.go` files modified ✅
- 5 consecutive `go test -race ./execution/engineapi/...` runs all pass
✅

CI job reference:
https://github.com/erigontech/erigon/actions/runs/23983504381/job/69951584262

---------

Co-authored-by: erigon-copilot[bot] <erigon-copilot[bot]@users.noreply.github.com>
Co-authored-by: Giulio Rebuffo <giulio.rebuffo@gmail.com>
Co-authored-by: Alexey Sharov <AskAlexSharov@gmail.com>
Co-authored-by: Matt Joiner <anacrolix@gmail.com>
Co-authored-by: yperbasis <andrey.ashikhmin@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com>

Author: 6 people

cmd/rpcdaemon/cli/config.go

@@ -810,6 +810,7 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 		}
 		listener, httpAddr, err := node.StartHTTPEndpoint(httpEndpoint, &node.HttpEndpointConfig{
 			Timeouts: cfg.HTTPTimeouts,
+			Listener: cfg.HttpListener,
 		}, apiHandler)
 		if err != nil {
 			return fmt.Errorf("could not start RPC api: %w", err)
@@ -995,6 +996,7 @@ func createEngineListener(cfg *httpcfg.HttpCfg, engineApi []rpc.API, logger log.
 
 	engineListener, engineAddr, err := node.StartHTTPEndpoint(engineHttpEndpoint, &node.HttpEndpointConfig{
 		Timeouts: cfg.AuthRpcTimeouts,
+		Listener: cfg.AuthRpcListener,
 	}, engineApiHandler)
 	if err != nil {
 		return nil, nil, "", fmt.Errorf("could not start RPC api: %w", err)

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -17,6 +17,7 @@
 package httpcfg
 
 import (
+	"net"
 	"time"
 
 	"github.com/erigontech/erigon/db/datadir"
@@ -114,4 +115,9 @@ type HttpCfg struct {
 
 	RpcTxSyncDefaultTimeout time.Duration // Default timeout for eth_sendRawTransactionSync
 	RpcTxSyncMaxTimeout     time.Duration // Maximum timeout for eth_sendRawTransactionSync
+
+	// Pre-created listeners for testing (avoids TOCTOU port races).
+	// When set, these listeners are passed to StartHTTPEndpoint instead of binding a new port.
+	HttpListener    net.Listener
+	AuthRpcListener net.Listener
 }

execution/engineapi/engineapitester/engine_api_tester.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net"
 	"path"
 	"testing"
 	"time"
@@ -33,7 +34,6 @@ import (
 	"github.com/erigontech/erigon/cmd/rpcdaemon/cli/httpcfg"
 	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/crypto"
-	"github.com/erigontech/erigon/common/freeport"
 	"github.com/erigontech/erigon/common/hexutil"
 	"github.com/erigontech/erigon/common/log/v3"
 	"github.com/erigontech/erigon/common/testlog"
@@ -117,28 +117,45 @@ func DefaultEngineApiTesterGenesis(t *testing.T) (*types.Genesis, *ecdsa.Private
 	return genesis, coinbasePrivKey
 }
 
+// localhostEphemeral asks the kernel to pick a free TCP port on the loopback
+// interface. Used by the engine-api test harness to avoid TOCTOU races with
+// concurrent tests selecting the same port.
+const localhostEphemeral = "127.0.0.1:0"
+
 func InitialiseEngineApiTester(t testing.TB, args EngineApiTesterInitArgs) EngineApiTester {
 	ctx := t.Context()
 	logger := args.Logger
 	dirs := datadir.New(args.DataDir)
 	genesis := args.Genesis
-	sentryPort, err := freeport.NextFreePort()
-	require.NoError(t, err)
-	engineApiPort, err := freeport.NextFreePort()
+
+	// Pre-bind the HTTP listeners on a kernel-assigned port to avoid TOCTOU
+	// port races with concurrent tests. Each listener is held open until the
+	// matching server takes ownership of the socket. The t.Cleanup calls are
+	// safety nets if InitialiseEngineApiTester aborts before hand-off; on the
+	// happy path the http server's Shutdown closes the listener first and the
+	// cleanup is a silent no-op. The sentry/P2P stack picks its own kernel-
+	// assigned port directly via its config below, so no pre-bind there.
+	jsonRpcListener, err := net.Listen("tcp", localhostEphemeral)
 	require.NoError(t, err)
-	jsonRpcPort, err := freeport.NextFreePort()
+	t.Cleanup(func() { _ = jsonRpcListener.Close() })
+	jsonRpcPort := jsonRpcListener.Addr().(*net.TCPAddr).Port
+	engineApiListener, err := net.Listen("tcp", localhostEphemeral)
 	require.NoError(t, err)
-	logger.Debug("[engine-api-tester] selected ports", "sentry", sentryPort, "engineApi", engineApiPort, "jsonRpc", jsonRpcPort)
+	t.Cleanup(func() { _ = engineApiListener.Close() })
+	engineApiPort := engineApiListener.Addr().(*net.TCPAddr).Port
+	logger.Debug("[engine-api-tester] selected ports", "engineApi", engineApiPort, "jsonRpc", jsonRpcPort)
 
 	httpConfig := httpcfg.HttpCfg{
 		Enabled:                  true,
 		HttpServerEnabled:        true,
 		WebsocketEnabled:         true,
 		HttpListenAddress:        "127.0.0.1",
 		HttpPort:                 jsonRpcPort,
+		HttpListener:             jsonRpcListener,
 		API:                      []string{"eth"},
 		AuthRpcHTTPListenAddress: "127.0.0.1",
 		AuthRpcPort:              engineApiPort,
+		AuthRpcListener:          engineApiListener,
 		JWTSecretPath:            path.Join(args.DataDir, "jwt.hex"),
 		ReturnDataLimit:          100_000,
 		EvmCallTimeout:           rpccfg.DefaultEvmCallTimeout,
@@ -155,13 +172,13 @@ func InitialiseEngineApiTester(t testing.TB, args EngineApiTesterInitArgs) Engin
 		Dirs: dirs,
 		Http: httpConfig,
 		P2P: p2p.Config{
-			ListenAddr:      fmt.Sprintf("127.0.0.1:%d", sentryPort),
+			ListenAddr:      localhostEphemeral,
 			MaxPeers:        1,
 			MaxPendingPeers: 1,
 			NoDiscovery:     true,
 			NoDial:          true,
 			ProtocolVersion: []uint{direct.ETH68},
-			AllowedPorts:    []uint{uint(sentryPort)},
+			AllowedPorts:    []uint{0},
 			PrivateKey:      nodeKey,
 		},
 	}

node/endpoints.go

@@ -42,6 +42,7 @@ type HttpEndpointConfig struct {
 	HTTPS    bool
 	CertFile string
 	KeyFile  string
+	Listener net.Listener // optional pre-created listener; if set, StartHTTPEndpoint uses it instead of creating a new one
 }
 
 // StartHTTPEndpoint starts the HTTP RPC endpoint.
@@ -51,12 +52,17 @@ func StartHTTPEndpoint(urlEndpoint string, cfg *HttpEndpointConfig, handler http
 		listener net.Listener
 		err      error
 	)
-	socketUrl, err := url.Parse(urlEndpoint)
-	if err != nil {
-		return nil, nil, fmt.Errorf("malformatted http listen url %s: %w", urlEndpoint, err)
-	}
-	if listener, err = net.Listen(socketUrl.Scheme, socketUrl.Host+socketUrl.EscapedPath()); err != nil {
-		return nil, nil, err
+	if cfg.Listener != nil {
+		listener = cfg.Listener
+	} else {
+		var socketUrl *url.URL
+		socketUrl, err = url.Parse(urlEndpoint)
+		if err != nil {
+			return nil, nil, fmt.Errorf("malformed http listen url %s: %w", urlEndpoint, err)
+		}
+		if listener, err = net.Listen(socketUrl.Scheme, socketUrl.Host+socketUrl.EscapedPath()); err != nil {
+			return nil, nil, err
+		}
 	}
 	// make sure timeout values are meaningful
 	CheckTimeouts(&cfg.Timeouts)