observer: Reduce memory usage (#8649)

Reuse shared HTTP clients and transports instead of initializing a new
client for every probe request, and close HTTP response bodies in CRL,
HTTP, and TLS probers.

These changes line up well with the heap and goroutine profiles, which
show `net/http` transport churn and a large number of
`net/http.(*persistConn).readLoop` / `writeLoop` goroutines
accumulating, consistent with connections and associated transport
buffers not being closed promptly.

Fixes #8650

Author: beautifulentropy

observer/obsclient/obsclient.go

@@ -7,8 +7,19 @@ import (
 	"net/http"
 )
 
-// Client returns an http.Client for use in probers.
+var secureClient = newClient(false)
+var insecureClient = newClient(true)
+
+// Client returns a shared *http.Client, with the appropriate TLS configuration,
+// shared across all probers.
 func Client(insecure bool) *http.Client {
+	if insecure {
+		return insecureClient
+	}
+	return secureClient
+}
+
+func newClient(insecure bool) *http.Client {
 	// Use the default transport, because it comes with useful defaults that are
 	// not just the http.Transport zero-values.
 	t := http.DefaultTransport.(*http.Transport).Clone()

observer/probers/crl/crl.go

@@ -45,6 +45,7 @@ func (p CRLProbe) Probe(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {

observer/probers/http/http.go

@@ -45,6 +45,7 @@ func (p HTTPProbe) Probe(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	if !slices.Contains(p.rcodes, resp.StatusCode) {
 		return fmt.Errorf("got HTTP status code %d, but want one of %#v", resp.StatusCode, p.rcodes)

observer/probers/tls/tls.go

@@ -83,6 +83,7 @@ func checkOCSP(ctx context.Context, cert, issuer *x509.Certificate, want int) (b
 	if err != nil {
 		return false, err
 	}
+	defer res.Body.Close()
 
 	output, err := io.ReadAll(res.Body)
 	if err != nil {