tls: add read deadline to containers/image registry connections

A stalled TLS connection to a container registry (e.g. quay.io) can
cause image pulls to hang indefinitely. The HTTP response body read
blocks forever in tls.Conn.Read with no timeout, starving the entire
pull pipeline and leaving pods stuck in ContainerCreating for hours.

Wrap the HTTP transport dialer with a deadlineConn that enforces a
5-minute read deadline via SetReadDeadline on every Read call. When
triggered, bodyReader treats the timeout the same as ECONNRESET and
attempts a Range-based reconnect to resume the download. Also add a
2-minute ResponseHeaderTimeout to the transport.

Ref: https://redhat.atlassian.net/browse/OCPBUGS-79544

Author: rphillips

image/docker/body_reader.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"math"
 	"math/rand/v2"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -17,6 +18,16 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// isRetryableNetworkError returns true for network errors that indicate
+// a stalled or interrupted connection that may be resolved by reconnecting.
+func isRetryableNetworkError(err error) bool {
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return true
+	}
+	return false
+}
+
 const (
 	// bodyReaderMinimumProgress is the minimum progress we consider a good reason to retry
 	bodyReaderMinimumProgress = 1 * 1024 * 1024
@@ -147,7 +158,7 @@ func (br *bodyReader) Read(p []byte) (int, error) {
 		br.lastSuccessTime = time.Now()
 		return n, err // Unlike the default: case, don’t log anything.
 
-	case errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET):
+	case errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) || isRetryableNetworkError(err):
 		originalErr := err
 		redactedURL := br.logURL.Redacted()
 		if err := br.errorIfNotReconnecting(originalErr, redactedURL); err != nil {

image/docker/docker_client.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -39,6 +40,14 @@ import (
 )
 
 const (
+	// registryReadTimeout is the maximum duration a single read from a registry
+	// connection can block without returning any data before we consider the
+	// connection stalled. This is enforced via SetReadDeadline on the underlying
+	// net.Conn. When triggered, the read returns a timeout error which
+	// bodyReader treats as a reconnectable condition (like ECONNRESET), allowing
+	// it to resume the download with a Range request.
+	registryReadTimeout = 5 * time.Minute
+
 	dockerHostname   = "docker.io"
 	dockerV1Hostname = "index.docker.io"
 	dockerRegistry   = "registry-1.docker.io"
@@ -963,6 +972,22 @@ func (bt *bearerToken) readFromHTTPResponseBody(res *http.Response) error {
 	return nil
 }
 
+// deadlineConn wraps a net.Conn to enforce a read deadline on every Read call.
+// If no data arrives within the configured readTimeout, the Read returns a
+// net.Error with Timeout() == true, which bodyReader treats as a reconnectable
+// condition and retries with a Range request.
+type deadlineConn struct {
+	net.Conn
+	readTimeout time.Duration
+}
+
+func (c *deadlineConn) Read(p []byte) (int, error) {
+	if err := c.Conn.SetReadDeadline(time.Now().Add(c.readTimeout)); err != nil {
+		return 0, err
+	}
+	return c.Conn.Read(p)
+}
+
 // detectPropertiesHelper performs the work of detectProperties which executes
 // it at most once.
 func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
@@ -973,6 +998,17 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
 	}
 	tr := tlsclientconfig.NewTransport()
 	tr.TLSClientConfig = c.tlsClientConfig
+	// Wrap the dialer to set per-read deadlines on the underlying connection.
+	// This ensures that a stalled registry response (e.g. TLS read that never
+	// returns data) will time out instead of blocking indefinitely.
+	originalDial := tr.DialContext
+	tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := originalDial(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+		return &deadlineConn{Conn: conn, readTimeout: registryReadTimeout}, nil
+	}
 	// if set DockerProxyURL explicitly, use the DockerProxyURL instead of system proxy
 	if c.sys != nil && c.sys.DockerProxyURL != nil {
 		tr.Proxy = http.ProxyURL(c.sys.DockerProxyURL)
@@ -982,6 +1018,7 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
 			return c.sys.DockerProxy(request.URL)
 		}
 	}
+	tr.ResponseHeaderTimeout = 2 * time.Minute
 	c.client = &http.Client{Transport: tr}
 
 	ping := func(scheme string) error {