CRE-4405: Add request tracing, add default transport to outgoing HTTP requests

Author: cedric-cordenier

core/cmd/shell.go

@@ -51,6 +51,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/ccv/ccvcommon"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/cre"
+	gatewaynetwork "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
 	"github.com/smartcontractkit/chainlink/v2/core/services/llo"
 	"github.com/smartcontractkit/chainlink/v2/core/services/llo/retirement"
@@ -80,6 +81,7 @@ func metricViews() []sdkmetric.View {
 		ccvcommon.MetricViews(),
 		ocr3beholderwrapper.MetricViews(),
 		ocr3_1beholderwrapper.MetricViews(),
+		gatewaynetwork.HTTPClientMetricViews(),
 	)
 }
 

core/scripts/go.mod

@@ -648,6 +648,8 @@ require (
 
 replace github.com/fbsobreira/gotron-sdk => github.com/smartcontractkit/chainlink-tron/relayer/gotron-sdk v0.0.5-0.20260218133534-cbd44da2856b
 
+replace github.com/doyensec/safeurl => github.com/cedric-cordenier/safeurl v0.0.0-20260519102940-15f2ee3f8b50
+
 // moby/go-archive v0.2.0 removed deprecated aliases (archive.Gzip, etc.) that
 // docker/cli@v28.5.x still uses. docker/compose has not migrated to docker/cli v29
 // yet, so we pin to v0.1.0 which has both the old aliases and the new compression API.

core/scripts/go.sum

@@ -8,6 +8,7 @@ import (
 	"io"
 	"maps"
 	"net/http"
+	"net/http/httptrace"
 	"slices"
 	"strings"
 
@@ -169,9 +170,10 @@ func responseHeadersFromNetHeader(h http.Header) (map[string]string, map[string]
 }
 
 type httpClient struct {
-	client *safeurl.WrappedClient
-	config HTTPClientConfig
-	lggr   logger.Logger
+	client  *safeurl.WrappedClient
+	config  HTTPClientConfig
+	lggr    logger.Logger
+	metrics *httpClientMetrics
 }
 
 // NewHTTPClient creates a new NewHTTPClient
@@ -194,12 +196,21 @@ func NewHTTPClient(config HTTPClientConfig, lggr logger.Logger) (HTTPClient, err
 		SetBlockedIPs(config.BlockedIPs...).
 		SetBlockedIPsCIDR(config.BlockedIPsCIDR...).
 		SetCheckRedirect(disableRedirects).
+		SetTransport(
+			http.DefaultTransport.(*http.Transport),
+		).
 		Build()
 
+	metrics, err := newHTTPClientMetrics()
+	if err != nil {
+		return nil, err
+	}
+
 	return &httpClient{
-		config: config,
-		client: safeurl.Client(safeConfig),
-		lggr:   lggr,
+		config:  config,
+		client:  safeurl.Client(safeConfig),
+		lggr:    lggr,
+		metrics: metrics,
 	}, nil
 }
 
@@ -297,8 +308,13 @@ func (c *httpClient) Send(ctx context.Context, req HTTPRequest) (*HTTPResponse,
 	timeoutCtx, cancel := context.WithTimeout(ctx, to)
 	defer cancel()
 
+	requestStart := time.Now()
+	trace, traceState := newClientTrace(ctx, req.Method, requestStart, c.metrics)
+	timeoutCtx = httptrace.WithClientTrace(timeoutCtx, trace)
+
 	r, err := http.NewRequestWithContext(timeoutCtx, req.Method, req.URL, bytes.NewBuffer(req.Body))
 	if err != nil {
+		c.metrics.recordTotal(ctx, req.Method, 0, false, false, time.Since(requestStart))
 		return nil, err
 	}
 	for k, values := range requestToNetHeader(req) {
@@ -309,6 +325,7 @@ func (c *httpClient) Send(ctx context.Context, req HTTPRequest) (*HTTPResponse,
 
 	resp, err := c.client.Do(r)
 	if err != nil {
+		c.metrics.recordTotal(ctx, req.Method, 0, false, traceState.connReused.Load(), time.Since(requestStart))
 		if isBlockedRequest(err) {
 			c.lggr.Warnw("HTTP request blocked", "err", err)
 			return nil, fmt.Errorf("%w: %w", ErrBlockedRequest, err)
@@ -324,10 +341,13 @@ func (c *httpClient) Send(ctx context.Context, req HTTPRequest) (*HTTPResponse,
 	reader := http.MaxBytesReader(nil, resp.Body, int64(n))
 	body, err := io.ReadAll(reader)
 	if err != nil {
+		c.metrics.recordTotal(ctx, req.Method, resp.StatusCode, false, traceState.connReused.Load(), time.Since(requestStart))
 		c.lggr.Errorw("failed to read HTTP response body", "err", err)
 		return nil, errors.Join(err, ErrHTTPRead)
 	}
 
+	c.metrics.recordTotal(ctx, req.Method, resp.StatusCode, true, traceState.connReused.Load(), time.Since(requestStart))
+
 	headers, multiHeaders := responseHeadersFromNetHeader(resp.Header)
 	c.lggr.Debugw("received HTTP response", "statusCode", resp.StatusCode)
 	return &HTTPResponse{

core/services/gateway/network/httpclient.go

@@ -431,6 +431,8 @@ require (
 
 replace github.com/fbsobreira/gotron-sdk => github.com/smartcontractkit/chainlink-tron/relayer/gotron-sdk v0.0.5-0.20260218133534-cbd44da2856b
 
+replace github.com/doyensec/safeurl => github.com/cedric-cordenier/safeurl v0.0.0-20260519102940-15f2ee3f8b50
+
 tool github.com/smartcontractkit/chainlink-common/pkg/loop/cmd/loopinstall
 
 tool github.com/smartcontractkit/chainlink-common/script/cmd/dependabot