From f19e5c8d8956f83959dc5a74cd9732ec3dc68a68 Mon Sep 17 00:00:00 2001 From: Kiel C Date: Wed, 1 Jul 2026 16:18:26 +0000 Subject: [PATCH 1/4] Apply LimitedReader structs to all io.ReadAll calls. --- bdns/dns.go | 2 +- bdns/dns_test.go | 2 +- cmd/crl-checker/main.go | 2 +- cmd/shell.go | 2 +- crl/checker/checker_test.go | 4 ++-- crl/storer/storer.go | 2 +- crl/storer/storer_test.go | 2 +- linter/pkimetal/client.go | 2 +- observer/probers/aia/aia.go | 2 +- observer/probers/crl/crl.go | 2 +- observer/probers/tls/tls.go | 4 ++-- salesforce/pardot.go | 5 +++-- sfe/zendesk/zendesk.go | 2 +- test/chall-test-srv-client/client.go | 2 +- test/chall-test-srv/http.go | 2 +- test/checkari/main.go | 4 ++-- test/ct-test-srv/main.go | 4 ++-- test/integration/crl_test.go | 6 +++--- test/integration/errors_test.go | 2 +- test/integration/otel_test.go | 2 +- test/integration/revocation_test.go | 2 +- test/integration/wfe_test.go | 2 +- test/load-generator/acme/directory.go | 2 +- test/load-generator/boulder-calls.go | 14 +++++++------- test/load-generator/state.go | 2 +- test/s3-test-srv/main.go | 2 +- test/salesforce-test-srv/main.go | 2 +- test/zendeskfake/zendeskfake_test.go | 2 +- tools/crldps/main.go | 2 +- wfe2/wfe_test.go | 2 +- 30 files changed, 44 insertions(+), 43 deletions(-) diff --git a/bdns/dns.go b/bdns/dns.go index bb147bb2d32..cbb9ffda1b0 100644 --- a/bdns/dns.go +++ b/bdns/dns.go @@ -379,7 +379,7 @@ func (d *dohExchanger) ExchangeContext(ctx context.Context, query *dns.Msg, serv return nil, d.clk.Since(start), fmt.Errorf("doh: http status %d", resp.StatusCode) } - b, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, d.clk.Since(start), fmt.Errorf("doh: reading response body: %w", err) } diff --git a/bdns/dns_test.go b/bdns/dns_test.go index 5c083d25155..e3c7c6329e2 100644 --- a/bdns/dns_test.go +++ b/bdns/dns_test.go @@ -39,7 +39,7 @@ func mockDNSQuery(w http.ResponseWriter, httpReq *http.Request) { fmt.Fprintf(w, "client didn't accept Content-Type: application/dns-message") } - requestBody, err := io.ReadAll(httpReq.Body) + requestBody, err := io.ReadAll(&io.LimitedReader{R: httpReq.Body, N: 100_000_000}) if err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Fprintf(w, "reading body: %s", err) diff --git a/cmd/crl-checker/main.go b/cmd/crl-checker/main.go index c3398caec2c..86edc6bc302 100644 --- a/cmd/crl-checker/main.go +++ b/cmd/crl-checker/main.go @@ -32,7 +32,7 @@ func downloadShard(url string) (*x509.RevocationList, error) { return nil, fmt.Errorf("downloading crl: http status %d", resp.StatusCode) } - crlBytes, err := io.ReadAll(resp.Body) + crlBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, fmt.Errorf("reading CRL bytes: %w", err) } diff --git a/cmd/shell.go b/cmd/shell.go index 0cfa6fff2df..9ba2dd08afc 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -511,7 +511,7 @@ func ValidateYAMLConfig(cv *ConfigValidator, in io.Reader) error { // Register custom types for use with existing validation tags. validate.RegisterCustomTypeFunc(config.DurationCustomTypeFunc, config.Duration{}) - inBytes, err := io.ReadAll(in) + inBytes, err := io.ReadAll(&io.LimitedReader{R: in, N: 100_000_000}) if err != nil { return err } diff --git a/crl/checker/checker_test.go b/crl/checker/checker_test.go index 53fc507f219..452c8f6145b 100644 --- a/crl/checker/checker_test.go +++ b/crl/checker/checker_test.go @@ -20,7 +20,7 @@ import ( func TestValidate(t *testing.T) { crlFile, err := os.Open("../../test/hierarchy/int-e1.crl.pem") test.AssertNotError(t, err, "opening test crl file") - crlPEM, err := io.ReadAll(crlFile) + crlPEM, err := io.ReadAll(&io.LimitedReader{R: crlFile, N: 100_000_000}) test.AssertNotError(t, err, "reading test crl file") crlDER, _ := pem.Decode(crlPEM) crl, err := x509.ParseRevocationList(crlDER.Bytes) @@ -43,7 +43,7 @@ func TestValidate(t *testing.T) { crlFile, err = os.Open("../../linter/lints/cabf_br/testdata/crl_long_validity.pem") test.AssertNotError(t, err, "opening test crl file") - crlPEM, err = io.ReadAll(crlFile) + crlPEM, err = io.ReadAll(&io.LimitedReader{R: crlFile, N: 100_000_000}) test.AssertNotError(t, err, "reading test crl file") crlDER, _ = pem.Decode(crlPEM) crl, err = x509.ParseRevocationList(crlDER.Bytes) diff --git a/crl/storer/storer.go b/crl/storer/storer.go index 6455d77bdf9..97c69d0a9ca 100644 --- a/crl/storer/storer.go +++ b/crl/storer/storer.go @@ -181,7 +181,7 @@ func (cs *crlStorer) UploadCRL(stream grpc.ClientStreamingServer[cspb.UploadCRLR cs.log.Info(ctx, "Proceeding because no previous CRL found") } else { defer prevObj.Body.Close() - prevBytes, err := io.ReadAll(prevObj.Body) + prevBytes, err := io.ReadAll(&io.LimitedReader{R: prevObj.Body, N: 100_000_000}) if err != nil { return fmt.Errorf("downloading previous CRL for %s: %w", crlId, err) } diff --git a/crl/storer/storer_test.go b/crl/storer/storer_test.go index 6285370b52b..ef522cc884f 100644 --- a/crl/storer/storer_test.go +++ b/crl/storer/storer_test.go @@ -283,7 +283,7 @@ type fakeSimpleS3 struct { } func (p *fakeSimpleS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { - recvBytes, err := io.ReadAll(params.Body) + recvBytes, err := io.ReadAll(&io.LimitedReader{R: params.Body, N: 100_000_000}) if err != nil { return nil, err } diff --git a/linter/pkimetal/client.go b/linter/pkimetal/client.go index 4637e6a56f2..5bcef734a0b 100644 --- a/linter/pkimetal/client.go +++ b/linter/pkimetal/client.go @@ -80,7 +80,7 @@ func (pkim *Client) Execute(endpoint string, der []byte) (*lint.LintResult, erro return nil, fmt.Errorf("got status %d (%s) from pkimetal API", resp.StatusCode, resp.Status) } - resJSON, err := io.ReadAll(resp.Body) + resJSON, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, fmt.Errorf("reading response from pkimetal API: %s", err) } diff --git a/observer/probers/aia/aia.go b/observer/probers/aia/aia.go index c6c90ed6db5..ff381e1a774 100644 --- a/observer/probers/aia/aia.go +++ b/observer/probers/aia/aia.go @@ -48,7 +48,7 @@ func (p AIAProbe) Probe(ctx context.Context) error { return fmt.Errorf("certificate Content-Type is %q but want application/pkix-cert", contentType) } - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return err } diff --git a/observer/probers/crl/crl.go b/observer/probers/crl/crl.go index 80e8d6f7982..bdc520b61e9 100644 --- a/observer/probers/crl/crl.go +++ b/observer/probers/crl/crl.go @@ -47,7 +47,7 @@ func (p CRLProbe) Probe(ctx context.Context) error { } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return err } diff --git a/observer/probers/tls/tls.go b/observer/probers/tls/tls.go index 7b3885a0530..f3b1e4ebdb3 100644 --- a/observer/probers/tls/tls.go +++ b/observer/probers/tls/tls.go @@ -85,7 +85,7 @@ func checkOCSP(ctx context.Context, cert, issuer *x509.Certificate, want int) (b } defer res.Body.Close() - output, err := io.ReadAll(res.Body) + output, err := io.ReadAll(&io.LimitedReader{R: res.Body, N: 100_000_000}) if err != nil { return false, err } @@ -114,7 +114,7 @@ func checkCRL(ctx context.Context, cert, issuer *x509.Certificate, want int) (bo } defer resp.Body.Close() - der, err := io.ReadAll(resp.Body) + der, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return false, fmt.Errorf("reading CRL: %w", err) } diff --git a/salesforce/pardot.go b/salesforce/pardot.go index f3852ef3475..4b582aa34c2 100644 --- a/salesforce/pardot.go +++ b/salesforce/pardot.go @@ -11,6 +11,7 @@ import ( "time" "github.com/jmhodges/clock" + "github.com/letsencrypt/boulder/core" ) @@ -117,7 +118,7 @@ func (pc *SalesforceClientImpl) updateToken() error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - body, readErr := io.ReadAll(resp.Body) + body, readErr := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if readErr != nil { return fmt.Errorf("token request failed with status %d; while reading body: %w", resp.StatusCode, readErr) } @@ -202,7 +203,7 @@ func (pc *SalesforceClientImpl) SendContact(email string) error { return nil } - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) resp.Body.Close() if err != nil { diff --git a/sfe/zendesk/zendesk.go b/sfe/zendesk/zendesk.go index e67a2309157..bdb21c6a3ed 100644 --- a/sfe/zendesk/zendesk.go +++ b/sfe/zendesk/zendesk.go @@ -177,7 +177,7 @@ func (c *Client) doJSONRequest(method, reqURL string, body []byte) ([]byte, erro } defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) + respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, fmt.Errorf("failed to read zendesk response body: %w", err) } diff --git a/test/chall-test-srv-client/client.go b/test/chall-test-srv-client/client.go index 38938527c6f..da02a947979 100644 --- a/test/chall-test-srv-client/client.go +++ b/test/chall-test-srv-client/client.go @@ -73,7 +73,7 @@ func (c *Client) postURL(path string, body any) ([]byte, error) { if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, endpoint) } - respBytes, err := io.ReadAll(resp.Body) + respBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, fmt.Errorf("reading response from %s: %w", endpoint, err) } diff --git a/test/chall-test-srv/http.go b/test/chall-test-srv/http.go index 1c634cff04d..93d5463e831 100644 --- a/test/chall-test-srv/http.go +++ b/test/chall-test-srv/http.go @@ -11,7 +11,7 @@ import ( // and unmarshal it into the provided ob. If an error occurs at any point it // will be returned. func mustParsePOST(ob any, request *http.Request) error { - jsonBody, err := io.ReadAll(request.Body) + jsonBody, err := io.ReadAll(&io.LimitedReader{R: request.Body, N: 100_000_000}) if err != nil { return err } diff --git a/test/checkari/main.go b/test/checkari/main.go index ddea4caec05..3df0788b946 100644 --- a/test/checkari/main.go +++ b/test/checkari/main.go @@ -49,7 +49,7 @@ func createRequest(cert *x509.Certificate) ([]byte, error) { } func parseResponse(resp *http.Response) (*core.RenewalInfo, error) { - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, err } @@ -96,7 +96,7 @@ func getARIURL(directory string) (string, error) { } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return "", err } diff --git a/test/ct-test-srv/main.go b/test/ct-test-srv/main.go index 66a13cf42d4..32bb027c232 100644 --- a/test/ct-test-srv/main.go +++ b/test/ct-test-srv/main.go @@ -45,7 +45,7 @@ func readJSON(r *http.Request, output any) error { if r.Method != "POST" { return fmt.Errorf("incorrect method; only POST allowed") } - bodyBytes, err := io.ReadAll(r.Body) + bodyBytes, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) if err != nil { return err } @@ -121,7 +121,7 @@ func (is *integrationSrv) addChainOrPre(w http.ResponseWriter, r *http.Request, http.NotFound(w, r) return } - bodyBytes, err := io.ReadAll(r.Body) + bodyBytes, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/test/integration/crl_test.go b/test/integration/crl_test.go index a18161bcc7f..ac49d592240 100644 --- a/test/integration/crl_test.go +++ b/test/integration/crl_test.go @@ -160,7 +160,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err := io.ReadAll(resp.Body) + reason, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() @@ -173,7 +173,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err = io.ReadAll(resp.Body) + reason, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() @@ -186,7 +186,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err = io.ReadAll(resp.Body) + reason, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() diff --git a/test/integration/errors_test.go b/test/integration/errors_test.go index 83eab5f71a4..a3ea8e45ced 100644 --- a/test/integration/errors_test.go +++ b/test/integration/errors_test.go @@ -221,7 +221,7 @@ func TestBadSignatureAlgorithm(t *testing.T) { } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { t.Fatalf("reading HTTP response: %s", err) } diff --git a/test/integration/otel_test.go b/test/integration/otel_test.go index bed380e3166..e622f89d4e1 100644 --- a/test/integration/otel_test.go +++ b/test/integration/otel_test.go @@ -72,7 +72,7 @@ func getTraceFromJaeger(t *testing.T, traceID trace.TraceID) Trace { } test.AssertEquals(t, resp.StatusCode, http.StatusOK) - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) test.AssertNotError(t, err, "failed to read trace body") var parsed TraceResponse diff --git a/test/integration/revocation_test.go b/test/integration/revocation_test.go index 8ae4b0c495e..9041d40dbac 100644 --- a/test/integration/revocation_test.go +++ b/test/integration/revocation_test.go @@ -108,7 +108,7 @@ func getCRL(t *testing.T, crlURL string, issuerCert *x509.Certificate) *x509.Rev if resp.StatusCode != http.StatusOK { t.Fatalf("fetching %s: status code %d", crlURL, resp.StatusCode) } - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { t.Fatalf("reading CRL from %s: %s", crlURL, err) } diff --git a/test/integration/wfe_test.go b/test/integration/wfe_test.go index ea50129d17d..c36539a59aa 100644 --- a/test/integration/wfe_test.go +++ b/test/integration/wfe_test.go @@ -49,7 +49,7 @@ func TestWFEHTTPMetrics(t *testing.T) { resp, err = http.Get("http://boulder.service.consul:8013/metrics") test.AssertNotError(t, err, "GET boulder-wfe2 metrics") test.AssertEquals(t, resp.StatusCode, http.StatusOK) - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) test.AssertNotError(t, err, "Reading boulder-wfe2 metrics response") test.AssertContains(t, string(body), `response_time_count{code="200",endpoint="/directory",method="GET"}`) resp.Body.Close() diff --git a/test/load-generator/acme/directory.go b/test/load-generator/acme/directory.go index 9be438b805a..7b80d0a2648 100644 --- a/test/load-generator/acme/directory.go +++ b/test/load-generator/acme/directory.go @@ -150,7 +150,7 @@ func getRawDirectory(directoryURL string) ([]byte, error) { return nil, ErrInvalidDirectoryHTTPCode } - rawDirectory, err := io.ReadAll(resp.Body) + rawDirectory, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, err } diff --git a/test/load-generator/boulder-calls.go b/test/load-generator/boulder-calls.go index c395a6ee3d6..1ddc9a7a74e 100644 --- a/test/load-generator/boulder-calls.go +++ b/test/load-generator/boulder-calls.go @@ -198,7 +198,7 @@ func newOrder(s *State, c *acmeCache) error { return fmt.Errorf("%s, post failed: %s", newOrderURL, err) } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return fmt.Errorf("%s, bad response: %s", newOrderURL, body) } @@ -243,7 +243,7 @@ func getAuthorization(s *State, c *acmeCache, url string) (*core.Authorization, // Read the response body defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, err } @@ -322,7 +322,7 @@ func completeAuthorization(authz *core.Authorization, url string, s *State, c *a // Read the response body and cleanup when finished defer resp.Body.Close() - _, err = io.ReadAll(resp.Body) + _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return err } @@ -410,7 +410,7 @@ func getOrder(s *State, c *acmeCache, url string) (*OrderJSON, error) { } // Read the response body defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return nil, fmt.Errorf("%s, bad response: %s", url, body) } @@ -529,7 +529,7 @@ func finalizeOrder(s *State, c *acmeCache) error { defer resp.Body.Close() // Read the body to ensure there isn't an error. We don't need the actual // contents. - _, err = io.ReadAll(resp.Body) + _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return err } @@ -583,7 +583,7 @@ func getCert(s *State, c *acmeCache, url string) ([]byte, error) { return nil, fmt.Errorf("%s bad response: %s", url, err) } defer resp.Body.Close() - return io.ReadAll(resp.Body) + return io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) } // revokeCertificate removes a certificate url from the context, retrieves it, @@ -639,7 +639,7 @@ func revokeCertificate(s *State, c *acmeCache) error { } defer resp.Body.Close() - _, err = io.ReadAll(resp.Body) + _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return err } diff --git a/test/load-generator/state.go b/test/load-generator/state.go index 6d075740726..f173a41e12c 100644 --- a/test/load-generator/state.go +++ b/test/load-generator/state.go @@ -240,7 +240,7 @@ func (s *State) Restore(filename string) error { return err } - content, err := io.ReadAll(f) + content, err := io.ReadAll(&io.LimitedReader{R: f, N: 100_000_000}) if err != nil { return err } diff --git a/test/s3-test-srv/main.go b/test/s3-test-srv/main.go index 60f8e7a1c27..6cdf674695c 100644 --- a/test/s3-test-srv/main.go +++ b/test/s3-test-srv/main.go @@ -33,7 +33,7 @@ func (srv *s3TestSrv) handleS3(w http.ResponseWriter, r *http.Request) { } func (srv *s3TestSrv) handleUpload(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) + body, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("failed to read request body")) diff --git a/test/salesforce-test-srv/main.go b/test/salesforce-test-srv/main.go index ad007fdeaaf..38486f3f2a4 100644 --- a/test/salesforce-test-srv/main.go +++ b/test/salesforce-test-srv/main.go @@ -101,7 +101,7 @@ func (ts *testServer) upsertContactsHandler(w http.ResponseWriter, r *http.Reque return } - body, err := io.ReadAll(r.Body) + body, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) if err != nil { http.Error(w, "Failed to read request body", http.StatusInternalServerError) return diff --git a/test/zendeskfake/zendeskfake_test.go b/test/zendeskfake/zendeskfake_test.go index 0f7cfa89af3..8e89a2284e7 100644 --- a/test/zendeskfake/zendeskfake_test.go +++ b/test/zendeskfake/zendeskfake_test.go @@ -70,7 +70,7 @@ func doJSON(t *testing.T, method, urlStr, authHeader string, body []byte, setCon return nil, nil } - respBody, err := io.ReadAll(resp.Body) + respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { t.Errorf("reading response body for %s %s failed: %s", method, urlStr, err) err = resp.Body.Close() diff --git a/tools/crldps/main.go b/tools/crldps/main.go index cf64d1de08c..7e3e2afe354 100644 --- a/tools/crldps/main.go +++ b/tools/crldps/main.go @@ -126,7 +126,7 @@ func fetchAndCheck(crldp string, client http.Client, issuer *x509.Certificate) e return fmt.Errorf("unexpected status code while downloading crl: %s", http.StatusText(resp.StatusCode)) } - crlDer, err := io.ReadAll(resp.Body) + crlDer, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { return fmt.Errorf("error reading crl: %s", err) } diff --git a/wfe2/wfe_test.go b/wfe2/wfe_test.go index 38501994122..03567834f1f 100644 --- a/wfe2/wfe_test.go +++ b/wfe2/wfe_test.go @@ -2563,7 +2563,7 @@ func TestGetCertificateHEADHasCorrectBodyLength(t *testing.T) { if err != nil { test.AssertNotError(t, err, "do error") } - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) if err != nil { test.AssertNotEquals(t, err, "readall error") } From e18eb1d4689a843a05b3bea0a91b65c12f9d7ced Mon Sep 17 00:00:00 2001 From: Kiel C Date: Thu, 2 Jul 2026 18:14:36 +0000 Subject: [PATCH 2/4] Change reader sizes, remove LimitedReader from tests and utilities. Reduce to still very generous ~300K normal reader size. Increase to ~1G CRL reader size. --- bdns/dns.go | 2 +- bdns/dns_test.go | 2 +- cmd/crl-checker/main.go | 2 +- cmd/shell.go | 2 +- crl/checker/checker_test.go | 4 ++-- crl/storer/storer.go | 2 +- crl/storer/storer_test.go | 2 +- linter/pkimetal/client.go | 2 +- observer/probers/aia/aia.go | 2 +- observer/probers/ccadb/retryhttp.go | 2 +- observer/probers/crl/crl.go | 2 +- observer/probers/tls/tls.go | 4 ++-- salesforce/pardot.go | 4 ++-- sfe/zendesk/zendesk.go | 2 +- test/chall-test-srv-client/client.go | 2 +- test/chall-test-srv/http.go | 2 +- test/checkari/main.go | 4 ++-- test/ct-test-srv/main.go | 4 ++-- test/integration/crl_test.go | 6 +++--- test/integration/errors_test.go | 2 +- test/integration/otel_test.go | 2 +- test/integration/revocation_test.go | 2 +- test/integration/wfe_test.go | 2 +- test/load-generator/acme/directory.go | 2 +- test/load-generator/boulder-calls.go | 14 +++++++------- test/load-generator/state.go | 2 +- test/s3-test-srv/main.go | 2 +- test/salesforce-test-srv/main.go | 2 +- test/zendeskfake/zendeskfake_test.go | 2 +- tools/crldps/main.go | 2 +- wfe2/wfe_test.go | 2 +- 31 files changed, 44 insertions(+), 44 deletions(-) diff --git a/bdns/dns.go b/bdns/dns.go index cbb9ffda1b0..e00e0432309 100644 --- a/bdns/dns.go +++ b/bdns/dns.go @@ -379,7 +379,7 @@ func (d *dohExchanger) ExchangeContext(ctx context.Context, query *dns.Msg, serv return nil, d.clk.Since(start), fmt.Errorf("doh: http status %d", resp.StatusCode) } - b, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + b, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if err != nil { return nil, d.clk.Since(start), fmt.Errorf("doh: reading response body: %w", err) } diff --git a/bdns/dns_test.go b/bdns/dns_test.go index e3c7c6329e2..5c083d25155 100644 --- a/bdns/dns_test.go +++ b/bdns/dns_test.go @@ -39,7 +39,7 @@ func mockDNSQuery(w http.ResponseWriter, httpReq *http.Request) { fmt.Fprintf(w, "client didn't accept Content-Type: application/dns-message") } - requestBody, err := io.ReadAll(&io.LimitedReader{R: httpReq.Body, N: 100_000_000}) + requestBody, err := io.ReadAll(httpReq.Body) if err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Fprintf(w, "reading body: %s", err) diff --git a/cmd/crl-checker/main.go b/cmd/crl-checker/main.go index 86edc6bc302..4117590ae35 100644 --- a/cmd/crl-checker/main.go +++ b/cmd/crl-checker/main.go @@ -32,7 +32,7 @@ func downloadShard(url string) (*x509.RevocationList, error) { return nil, fmt.Errorf("downloading crl: http status %d", resp.StatusCode) } - crlBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + crlBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) if err != nil { return nil, fmt.Errorf("reading CRL bytes: %w", err) } diff --git a/cmd/shell.go b/cmd/shell.go index 9ba2dd08afc..cb8df16c50d 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -511,7 +511,7 @@ func ValidateYAMLConfig(cv *ConfigValidator, in io.Reader) error { // Register custom types for use with existing validation tags. validate.RegisterCustomTypeFunc(config.DurationCustomTypeFunc, config.Duration{}) - inBytes, err := io.ReadAll(&io.LimitedReader{R: in, N: 100_000_000}) + inBytes, err := io.ReadAll(&io.LimitedReader{R: in, N: 300_000}) if err != nil { return err } diff --git a/crl/checker/checker_test.go b/crl/checker/checker_test.go index 452c8f6145b..53fc507f219 100644 --- a/crl/checker/checker_test.go +++ b/crl/checker/checker_test.go @@ -20,7 +20,7 @@ import ( func TestValidate(t *testing.T) { crlFile, err := os.Open("../../test/hierarchy/int-e1.crl.pem") test.AssertNotError(t, err, "opening test crl file") - crlPEM, err := io.ReadAll(&io.LimitedReader{R: crlFile, N: 100_000_000}) + crlPEM, err := io.ReadAll(crlFile) test.AssertNotError(t, err, "reading test crl file") crlDER, _ := pem.Decode(crlPEM) crl, err := x509.ParseRevocationList(crlDER.Bytes) @@ -43,7 +43,7 @@ func TestValidate(t *testing.T) { crlFile, err = os.Open("../../linter/lints/cabf_br/testdata/crl_long_validity.pem") test.AssertNotError(t, err, "opening test crl file") - crlPEM, err = io.ReadAll(&io.LimitedReader{R: crlFile, N: 100_000_000}) + crlPEM, err = io.ReadAll(crlFile) test.AssertNotError(t, err, "reading test crl file") crlDER, _ = pem.Decode(crlPEM) crl, err = x509.ParseRevocationList(crlDER.Bytes) diff --git a/crl/storer/storer.go b/crl/storer/storer.go index 97c69d0a9ca..f73f3b3897a 100644 --- a/crl/storer/storer.go +++ b/crl/storer/storer.go @@ -181,7 +181,7 @@ func (cs *crlStorer) UploadCRL(stream grpc.ClientStreamingServer[cspb.UploadCRLR cs.log.Info(ctx, "Proceeding because no previous CRL found") } else { defer prevObj.Body.Close() - prevBytes, err := io.ReadAll(&io.LimitedReader{R: prevObj.Body, N: 100_000_000}) + prevBytes, err := io.ReadAll(&io.LimitedReader{R: prevObj.Body, N: 1_000_000_000}) if err != nil { return fmt.Errorf("downloading previous CRL for %s: %w", crlId, err) } diff --git a/crl/storer/storer_test.go b/crl/storer/storer_test.go index ef522cc884f..6285370b52b 100644 --- a/crl/storer/storer_test.go +++ b/crl/storer/storer_test.go @@ -283,7 +283,7 @@ type fakeSimpleS3 struct { } func (p *fakeSimpleS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { - recvBytes, err := io.ReadAll(&io.LimitedReader{R: params.Body, N: 100_000_000}) + recvBytes, err := io.ReadAll(params.Body) if err != nil { return nil, err } diff --git a/linter/pkimetal/client.go b/linter/pkimetal/client.go index 5bcef734a0b..4ed0a0e4e6c 100644 --- a/linter/pkimetal/client.go +++ b/linter/pkimetal/client.go @@ -80,7 +80,7 @@ func (pkim *Client) Execute(endpoint string, der []byte) (*lint.LintResult, erro return nil, fmt.Errorf("got status %d (%s) from pkimetal API", resp.StatusCode, resp.Status) } - resJSON, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + resJSON, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if err != nil { return nil, fmt.Errorf("reading response from pkimetal API: %s", err) } diff --git a/observer/probers/aia/aia.go b/observer/probers/aia/aia.go index ff381e1a774..c61e3d77d61 100644 --- a/observer/probers/aia/aia.go +++ b/observer/probers/aia/aia.go @@ -48,7 +48,7 @@ func (p AIAProbe) Probe(ctx context.Context) error { return fmt.Errorf("certificate Content-Type is %q but want application/pkix-cert", contentType) } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if err != nil { return err } diff --git a/observer/probers/ccadb/retryhttp.go b/observer/probers/ccadb/retryhttp.go index 2bacbfe2def..ec0438a7dcf 100644 --- a/observer/probers/ccadb/retryhttp.go +++ b/observer/probers/ccadb/retryhttp.go @@ -21,7 +21,7 @@ func getBody(ctx context.Context, url string) ([]byte, error) { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if err != nil { return nil, err } diff --git a/observer/probers/crl/crl.go b/observer/probers/crl/crl.go index bdc520b61e9..4111f6b7a7c 100644 --- a/observer/probers/crl/crl.go +++ b/observer/probers/crl/crl.go @@ -47,7 +47,7 @@ func (p CRLProbe) Probe(ctx context.Context) error { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) if err != nil { return err } diff --git a/observer/probers/tls/tls.go b/observer/probers/tls/tls.go index f3b1e4ebdb3..c718fbe28b6 100644 --- a/observer/probers/tls/tls.go +++ b/observer/probers/tls/tls.go @@ -85,7 +85,7 @@ func checkOCSP(ctx context.Context, cert, issuer *x509.Certificate, want int) (b } defer res.Body.Close() - output, err := io.ReadAll(&io.LimitedReader{R: res.Body, N: 100_000_000}) + output, err := io.ReadAll(&io.LimitedReader{R: res.Body, N: 300_000}) if err != nil { return false, err } @@ -114,7 +114,7 @@ func checkCRL(ctx context.Context, cert, issuer *x509.Certificate, want int) (bo } defer resp.Body.Close() - der, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + der, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) if err != nil { return false, fmt.Errorf("reading CRL: %w", err) } diff --git a/salesforce/pardot.go b/salesforce/pardot.go index 4b582aa34c2..e24ce32ff14 100644 --- a/salesforce/pardot.go +++ b/salesforce/pardot.go @@ -118,7 +118,7 @@ func (pc *SalesforceClientImpl) updateToken() error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - body, readErr := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, readErr := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if readErr != nil { return fmt.Errorf("token request failed with status %d; while reading body: %w", resp.StatusCode, readErr) } @@ -203,7 +203,7 @@ func (pc *SalesforceClientImpl) SendContact(email string) error { return nil } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) resp.Body.Close() if err != nil { diff --git a/sfe/zendesk/zendesk.go b/sfe/zendesk/zendesk.go index bdb21c6a3ed..46d0c5bc02a 100644 --- a/sfe/zendesk/zendesk.go +++ b/sfe/zendesk/zendesk.go @@ -177,7 +177,7 @@ func (c *Client) doJSONRequest(method, reqURL string, body []byte) ([]byte, erro } defer resp.Body.Close() - respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) if err != nil { return nil, fmt.Errorf("failed to read zendesk response body: %w", err) } diff --git a/test/chall-test-srv-client/client.go b/test/chall-test-srv-client/client.go index da02a947979..38938527c6f 100644 --- a/test/chall-test-srv-client/client.go +++ b/test/chall-test-srv-client/client.go @@ -73,7 +73,7 @@ func (c *Client) postURL(path string, body any) ([]byte, error) { if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, endpoint) } - respBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + respBytes, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading response from %s: %w", endpoint, err) } diff --git a/test/chall-test-srv/http.go b/test/chall-test-srv/http.go index 93d5463e831..1c634cff04d 100644 --- a/test/chall-test-srv/http.go +++ b/test/chall-test-srv/http.go @@ -11,7 +11,7 @@ import ( // and unmarshal it into the provided ob. If an error occurs at any point it // will be returned. func mustParsePOST(ob any, request *http.Request) error { - jsonBody, err := io.ReadAll(&io.LimitedReader{R: request.Body, N: 100_000_000}) + jsonBody, err := io.ReadAll(request.Body) if err != nil { return err } diff --git a/test/checkari/main.go b/test/checkari/main.go index 3df0788b946..ddea4caec05 100644 --- a/test/checkari/main.go +++ b/test/checkari/main.go @@ -49,7 +49,7 @@ func createRequest(cert *x509.Certificate) ([]byte, error) { } func parseResponse(resp *http.Response) (*core.RenewalInfo, error) { - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -96,7 +96,7 @@ func getARIURL(directory string) (string, error) { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { return "", err } diff --git a/test/ct-test-srv/main.go b/test/ct-test-srv/main.go index 32bb027c232..66a13cf42d4 100644 --- a/test/ct-test-srv/main.go +++ b/test/ct-test-srv/main.go @@ -45,7 +45,7 @@ func readJSON(r *http.Request, output any) error { if r.Method != "POST" { return fmt.Errorf("incorrect method; only POST allowed") } - bodyBytes, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { return err } @@ -121,7 +121,7 @@ func (is *integrationSrv) addChainOrPre(w http.ResponseWriter, r *http.Request, http.NotFound(w, r) return } - bodyBytes, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/test/integration/crl_test.go b/test/integration/crl_test.go index ac49d592240..a18161bcc7f 100644 --- a/test/integration/crl_test.go +++ b/test/integration/crl_test.go @@ -160,7 +160,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + reason, err := io.ReadAll(resp.Body) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() @@ -173,7 +173,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + reason, err = io.ReadAll(resp.Body) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() @@ -186,7 +186,7 @@ func TestCRLPipeline(t *testing.T) { resp, err = http.Get("http://localhost:4501/query?serial=" + serial) test.AssertNotError(t, err, "s3-test-srv GET /query failed") test.AssertEquals(t, resp.StatusCode, 200) - reason, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + reason, err = io.ReadAll(resp.Body) test.AssertNotError(t, err, "reading revocation reason") test.AssertEquals(t, string(reason), "5") resp.Body.Close() diff --git a/test/integration/errors_test.go b/test/integration/errors_test.go index a3ea8e45ced..83eab5f71a4 100644 --- a/test/integration/errors_test.go +++ b/test/integration/errors_test.go @@ -221,7 +221,7 @@ func TestBadSignatureAlgorithm(t *testing.T) { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading HTTP response: %s", err) } diff --git a/test/integration/otel_test.go b/test/integration/otel_test.go index e622f89d4e1..bed380e3166 100644 --- a/test/integration/otel_test.go +++ b/test/integration/otel_test.go @@ -72,7 +72,7 @@ func getTraceFromJaeger(t *testing.T, traceID trace.TraceID) Trace { } test.AssertEquals(t, resp.StatusCode, http.StatusOK) - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) test.AssertNotError(t, err, "failed to read trace body") var parsed TraceResponse diff --git a/test/integration/revocation_test.go b/test/integration/revocation_test.go index 9041d40dbac..8ae4b0c495e 100644 --- a/test/integration/revocation_test.go +++ b/test/integration/revocation_test.go @@ -108,7 +108,7 @@ func getCRL(t *testing.T, crlURL string, issuerCert *x509.Certificate) *x509.Rev if resp.StatusCode != http.StatusOK { t.Fatalf("fetching %s: status code %d", crlURL, resp.StatusCode) } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading CRL from %s: %s", crlURL, err) } diff --git a/test/integration/wfe_test.go b/test/integration/wfe_test.go index c36539a59aa..ea50129d17d 100644 --- a/test/integration/wfe_test.go +++ b/test/integration/wfe_test.go @@ -49,7 +49,7 @@ func TestWFEHTTPMetrics(t *testing.T) { resp, err = http.Get("http://boulder.service.consul:8013/metrics") test.AssertNotError(t, err, "GET boulder-wfe2 metrics") test.AssertEquals(t, resp.StatusCode, http.StatusOK) - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) test.AssertNotError(t, err, "Reading boulder-wfe2 metrics response") test.AssertContains(t, string(body), `response_time_count{code="200",endpoint="/directory",method="GET"}`) resp.Body.Close() diff --git a/test/load-generator/acme/directory.go b/test/load-generator/acme/directory.go index 7b80d0a2648..9be438b805a 100644 --- a/test/load-generator/acme/directory.go +++ b/test/load-generator/acme/directory.go @@ -150,7 +150,7 @@ func getRawDirectory(directoryURL string) ([]byte, error) { return nil, ErrInvalidDirectoryHTTPCode } - rawDirectory, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + rawDirectory, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/test/load-generator/boulder-calls.go b/test/load-generator/boulder-calls.go index 1ddc9a7a74e..c395a6ee3d6 100644 --- a/test/load-generator/boulder-calls.go +++ b/test/load-generator/boulder-calls.go @@ -198,7 +198,7 @@ func newOrder(s *State, c *acmeCache) error { return fmt.Errorf("%s, post failed: %s", newOrderURL, err) } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("%s, bad response: %s", newOrderURL, body) } @@ -243,7 +243,7 @@ func getAuthorization(s *State, c *acmeCache, url string) (*core.Authorization, // Read the response body defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -322,7 +322,7 @@ func completeAuthorization(authz *core.Authorization, url string, s *State, c *a // Read the response body and cleanup when finished defer resp.Body.Close() - _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + _, err = io.ReadAll(resp.Body) if err != nil { return err } @@ -410,7 +410,7 @@ func getOrder(s *State, c *acmeCache, url string) (*OrderJSON, error) { } // Read the response body defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("%s, bad response: %s", url, body) } @@ -529,7 +529,7 @@ func finalizeOrder(s *State, c *acmeCache) error { defer resp.Body.Close() // Read the body to ensure there isn't an error. We don't need the actual // contents. - _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + _, err = io.ReadAll(resp.Body) if err != nil { return err } @@ -583,7 +583,7 @@ func getCert(s *State, c *acmeCache, url string) ([]byte, error) { return nil, fmt.Errorf("%s bad response: %s", url, err) } defer resp.Body.Close() - return io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + return io.ReadAll(resp.Body) } // revokeCertificate removes a certificate url from the context, retrieves it, @@ -639,7 +639,7 @@ func revokeCertificate(s *State, c *acmeCache) error { } defer resp.Body.Close() - _, err = io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + _, err = io.ReadAll(resp.Body) if err != nil { return err } diff --git a/test/load-generator/state.go b/test/load-generator/state.go index f173a41e12c..6d075740726 100644 --- a/test/load-generator/state.go +++ b/test/load-generator/state.go @@ -240,7 +240,7 @@ func (s *State) Restore(filename string) error { return err } - content, err := io.ReadAll(&io.LimitedReader{R: f, N: 100_000_000}) + content, err := io.ReadAll(f) if err != nil { return err } diff --git a/test/s3-test-srv/main.go b/test/s3-test-srv/main.go index 6cdf674695c..60f8e7a1c27 100644 --- a/test/s3-test-srv/main.go +++ b/test/s3-test-srv/main.go @@ -33,7 +33,7 @@ func (srv *s3TestSrv) handleS3(w http.ResponseWriter, r *http.Request) { } func (srv *s3TestSrv) handleUpload(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) + body, err := io.ReadAll(r.Body) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("failed to read request body")) diff --git a/test/salesforce-test-srv/main.go b/test/salesforce-test-srv/main.go index 38486f3f2a4..ad007fdeaaf 100644 --- a/test/salesforce-test-srv/main.go +++ b/test/salesforce-test-srv/main.go @@ -101,7 +101,7 @@ func (ts *testServer) upsertContactsHandler(w http.ResponseWriter, r *http.Reque return } - body, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 100_000_000}) + body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request body", http.StatusInternalServerError) return diff --git a/test/zendeskfake/zendeskfake_test.go b/test/zendeskfake/zendeskfake_test.go index 8e89a2284e7..0f7cfa89af3 100644 --- a/test/zendeskfake/zendeskfake_test.go +++ b/test/zendeskfake/zendeskfake_test.go @@ -70,7 +70,7 @@ func doJSON(t *testing.T, method, urlStr, authHeader string, body []byte, setCon return nil, nil } - respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + respBody, err := io.ReadAll(resp.Body) if err != nil { t.Errorf("reading response body for %s %s failed: %s", method, urlStr, err) err = resp.Body.Close() diff --git a/tools/crldps/main.go b/tools/crldps/main.go index 7e3e2afe354..cf64d1de08c 100644 --- a/tools/crldps/main.go +++ b/tools/crldps/main.go @@ -126,7 +126,7 @@ func fetchAndCheck(crldp string, client http.Client, issuer *x509.Certificate) e return fmt.Errorf("unexpected status code while downloading crl: %s", http.StatusText(resp.StatusCode)) } - crlDer, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + crlDer, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("error reading crl: %s", err) } diff --git a/wfe2/wfe_test.go b/wfe2/wfe_test.go index 03567834f1f..38501994122 100644 --- a/wfe2/wfe_test.go +++ b/wfe2/wfe_test.go @@ -2563,7 +2563,7 @@ func TestGetCertificateHEADHasCorrectBodyLength(t *testing.T) { if err != nil { test.AssertNotError(t, err, "do error") } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 100_000_000}) + body, err := io.ReadAll(resp.Body) if err != nil { test.AssertNotEquals(t, err, "readall error") } From 25207736919530f6cd5f24f1b5d8c1bba3e1a364 Mon Sep 17 00:00:00 2001 From: Kiel C Date: Thu, 2 Jul 2026 18:25:37 +0000 Subject: [PATCH 3/4] ccadb/retryhttp.go is used for CRLs, so increase. --- observer/probers/ccadb/retryhttp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/observer/probers/ccadb/retryhttp.go b/observer/probers/ccadb/retryhttp.go index ec0438a7dcf..7484ded6993 100644 --- a/observer/probers/ccadb/retryhttp.go +++ b/observer/probers/ccadb/retryhttp.go @@ -21,7 +21,7 @@ func getBody(ctx context.Context, url string) ([]byte, error) { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) if err != nil { return nil, err } From ce0ca7fe2add19712855a9931ac8e956276026cf Mon Sep 17 00:00:00 2001 From: Kiel C Date: Wed, 29 Jul 2026 00:42:42 +0000 Subject: [PATCH 4/4] Introduce core.ErrOnLimitReader and DefaultMaxRead --- bdns/dns.go | 5 +++- cmd/crl-checker/main.go | 3 +- cmd/shell.go | 3 +- core/core_test.go | 37 ++++++++++++++++++++++++ core/util.go | 44 +++++++++++++++++++++++++++++ linter/pkimetal/client.go | 5 +++- observer/probers/aia/aia.go | 5 +++- observer/probers/ccadb/retryhttp.go | 23 +++++++++------ observer/probers/crl/crl.go | 4 ++- observer/probers/tls/tls.go | 7 +++-- salesforce/pardot.go | 9 ++++-- sfe/zendesk/zendesk.go | 5 +++- 12 files changed, 130 insertions(+), 20 deletions(-) diff --git a/bdns/dns.go b/bdns/dns.go index f781d1620ac..bc375af96a7 100644 --- a/bdns/dns.go +++ b/bdns/dns.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/letsencrypt/boulder/core" blog "github.com/letsencrypt/boulder/log" "github.com/letsencrypt/boulder/metrics" ) @@ -375,7 +376,9 @@ func (d *dohExchanger) ExchangeContext(ctx context.Context, query *dns.Msg, serv return nil, d.clk.Since(start), fmt.Errorf("doh: http status %d", resp.StatusCode) } - b, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + // DNS response over 65535 is malformed https://datatracker.ietf.org/doc/html/rfc8484#section-6 + lmr := core.ErrOnLimitReader(resp.Body, 65_535) + b, err := io.ReadAll(lmr) if err != nil { return nil, d.clk.Since(start), fmt.Errorf("doh: reading response body: %w", err) } diff --git a/cmd/crl-checker/main.go b/cmd/crl-checker/main.go index b99345a4f4c..be7d1d99741 100644 --- a/cmd/crl-checker/main.go +++ b/cmd/crl-checker/main.go @@ -28,7 +28,8 @@ func downloadShard(url string) (*x509.RevocationList, error) { return nil, fmt.Errorf("downloading crl: http status %d", resp.StatusCode) } - crlBytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) + lmr := core.ErrOnLimitReader(resp.Body, 1_000_000_000) + crlBytes, err := io.ReadAll(lmr) if err != nil { return nil, fmt.Errorf("reading CRL bytes: %w", err) } diff --git a/cmd/shell.go b/cmd/shell.go index ad9a76d1b78..f41e90f6f2f 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -504,7 +504,8 @@ func ValidateYAMLConfig(cv *ConfigValidator, in io.Reader) error { // Register custom types for use with existing validation tags. validate.RegisterCustomTypeFunc(config.DurationCustomTypeFunc, config.Duration{}) - inBytes, err := io.ReadAll(&io.LimitedReader{R: in, N: 300_000}) + lmr := core.ErrOnLimitReader(in, core.DefaultMaxRead) + inBytes, err := io.ReadAll(lmr) if err != nil { return err } diff --git a/core/core_test.go b/core/core_test.go index 389acd86b32..cc28f4b57de 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -3,6 +3,8 @@ package core import ( "encoding/base64" "encoding/json" + "io" + "strings" "testing" "github.com/go-jose/go-jose/v4" @@ -80,3 +82,38 @@ func TestFingerprint(t *testing.T) { t.Errorf("Incorrect SHA-256 fingerprint: %v", digest) } } + +func TestErrOnLimitReader(t *testing.T) { + // test a read where the limit is larger than the source + strReader := strings.NewReader("foo bar baz bot") + lmr := ErrOnLimitReader(strReader, 21) + + strOut01, err := io.ReadAll(lmr) + + // should return no err, and the full source + test.AssertNotError(t, err, "unexpectedly errored") + test.AssertEquals(t, string(strOut01), "foo bar baz bot") + + // test a read where the limit exactly matches source size + strReader = strings.NewReader("foo bar baz bot") + lmr = ErrOnLimitReader(strReader, 15) + + strOut02, err := io.ReadAll(lmr) + + // should return an error, and the full source + test.AssertError(t, err, "unexpectedly succeeded") + test.AssertEquals(t, err, ErrReaderLimitReached) + test.AssertEquals(t, string(strOut02), "foo bar baz bot") + + // test a read where the limit is lower than the source + strReader = strings.NewReader("foo bar baz bot") + lmr = ErrOnLimitReader(strReader, 9) + + strOut03, err := io.ReadAll(lmr) + + // should return an error, and a partial read result + test.AssertError(t, err, "unexpectedly succeeded") + test.AssertEquals(t, err, ErrReaderLimitReached) + test.AssertEquals(t, string(strOut03), "foo bar b") + +} diff --git a/core/util.go b/core/util.go index 97892b122d0..faeaaf72d21 100644 --- a/core/util.go +++ b/core/util.go @@ -51,6 +51,14 @@ var BuildHost string // BuildTime is set by the compiler and is used by GetBuildTime var BuildTime string +// DefaultMaxRead is for use by ErrOnLimitReader when to conveniently limit +// reads to less than half a MB, which should be most of the time +var DefaultMaxRead int64 = 300_000 + +// ErrReaderLimitReached as an exported error type allows callers to check for +// this error type after Read +var ErrReaderLimitReached error = errors.New("reader size limit reached") + func init() { expvar.NewString("BuildID").Set(BuildID) expvar.NewString("BuildTime").Set(BuildTime) @@ -431,3 +439,39 @@ func NormalizeIssuerDomainName(name string) (string, error) { } return name, nil } + +// An ErrOnLimitedReader reads from R but limits the amount of data returned to +// just N bytes. Each call to Read updates N to reflect the new amount +// remaining. +// Read returns ErrReaderLimitExceeded when N <= 0, or EOF when the underlying R +// returns EOF. +type ErrOnLimitedReader struct { + R io.Reader + N int64 +} + +// ErrOnLimitReader returns a Reader that reads from r but stops with +// ErrReaderLimitExceeded after n bytes. +// The underlying implementation is a *ErrOnLimitedReader. +// +// If LimitedReader gets an Err field, we can switch to use that +// https://github.com/golang/go/issues/51115 +func ErrOnLimitReader(r io.Reader, n int64) io.Reader { + return &ErrOnLimitedReader{r, n} +} + +func (ltdR *ErrOnLimitedReader) Read(b []byte) (n int, err error) { + // Important: when an ErrOnLimitedReader reads _exactly_ the limit amount, + // it results in the LimitReached Error + if ltdR.N <= 0 { + return 0, ErrReaderLimitReached + } + + if int64(len(b)) > ltdR.N { + b = b[0:ltdR.N] + } + + n, err = ltdR.R.Read(b) + ltdR.N -= int64(n) + return +} diff --git a/linter/pkimetal/client.go b/linter/pkimetal/client.go index 4ed0a0e4e6c..0818a3d4278 100644 --- a/linter/pkimetal/client.go +++ b/linter/pkimetal/client.go @@ -15,6 +15,8 @@ import ( "time" "github.com/zmap/zlint/v3/lint" + + "github.com/letsencrypt/boulder/core" ) // Config holds configuration for linting both certs and CRLs using PKIMetal. @@ -80,7 +82,8 @@ func (pkim *Client) Execute(endpoint string, der []byte) (*lint.LintResult, erro return nil, fmt.Errorf("got status %d (%s) from pkimetal API", resp.StatusCode, resp.Status) } - resJSON, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + resJSON, err := io.ReadAll(lmr) if err != nil { return nil, fmt.Errorf("reading response from pkimetal API: %s", err) } diff --git a/observer/probers/aia/aia.go b/observer/probers/aia/aia.go index c61e3d77d61..213c885399c 100644 --- a/observer/probers/aia/aia.go +++ b/observer/probers/aia/aia.go @@ -8,6 +8,8 @@ import ( "net/http" "github.com/prometheus/client_golang/prometheus" + + "github.com/letsencrypt/boulder/core" ) // AIAProbe is the exported 'Prober' object for monitors configured to @@ -48,7 +50,8 @@ func (p AIAProbe) Probe(ctx context.Context) error { return fmt.Errorf("certificate Content-Type is %q but want application/pkix-cert", contentType) } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + body, err := io.ReadAll(lmr) if err != nil { return err } diff --git a/observer/probers/ccadb/retryhttp.go b/observer/probers/ccadb/retryhttp.go index 7484ded6993..b2e81e4c843 100644 --- a/observer/probers/ccadb/retryhttp.go +++ b/observer/probers/ccadb/retryhttp.go @@ -6,6 +6,8 @@ import ( "io" "net/http" "time" + + "github.com/letsencrypt/boulder/core" ) func getBody(ctx context.Context, url string) ([]byte, error) { @@ -21,19 +23,24 @@ func getBody(ctx context.Context, url string) ([]byte, error) { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - // Truncate the response body in case it's too big to be useful in logs. - if len(body) > 400 { - body = body[:400] + // Read up to 400 bytes of response body to include in error logs + lmr := core.ErrOnLimitReader(resp.Body, 400) + body, err := io.ReadAll(lmr) + if err != nil && err != core.ErrReaderLimitReached { + return nil, err } + return nil, fmt.Errorf("http status %d for %q: %s", resp.StatusCode, url, string(body)) } + // Read up to ~1G of response body to return to caller + lmr := core.ErrOnLimitReader(resp.Body, 1_000_000_000) + body, err := io.ReadAll(lmr) + if err != nil { + return nil, err + } + return body, nil } diff --git a/observer/probers/crl/crl.go b/observer/probers/crl/crl.go index 4111f6b7a7c..ad67d235538 100644 --- a/observer/probers/crl/crl.go +++ b/observer/probers/crl/crl.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" + "github.com/letsencrypt/boulder/core" "github.com/letsencrypt/boulder/crl/idp" "github.com/letsencrypt/boulder/observer/obsclient" ) @@ -47,7 +48,8 @@ func (p CRLProbe) Probe(ctx context.Context) error { } defer resp.Body.Close() - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) + lmr := core.ErrOnLimitReader(resp.Body, 1_000_000_000) + body, err := io.ReadAll(lmr) if err != nil { return err } diff --git a/observer/probers/tls/tls.go b/observer/probers/tls/tls.go index 8bed2a39262..e1d5854d9c2 100644 --- a/observer/probers/tls/tls.go +++ b/observer/probers/tls/tls.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "golang.org/x/crypto/ocsp" + "github.com/letsencrypt/boulder/core" "github.com/letsencrypt/boulder/observer/obsclient" ) @@ -86,7 +87,8 @@ func checkOCSP(ctx context.Context, cert, issuer *x509.Certificate, want int) (b } defer res.Body.Close() - output, err := io.ReadAll(&io.LimitedReader{R: res.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(res.Body, core.DefaultMaxRead) + output, err := io.ReadAll(lmr) if err != nil { return false, err } @@ -115,7 +117,8 @@ func checkCRL(ctx context.Context, cert, issuer *x509.Certificate, want int) (bo } defer resp.Body.Close() - der, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1_000_000_000}) + lmr := core.ErrOnLimitReader(resp.Body, 1_000_000_000) + der, err := io.ReadAll(lmr) if err != nil { return false, fmt.Errorf("reading CRL: %w", err) } diff --git a/salesforce/pardot.go b/salesforce/pardot.go index e24ce32ff14..c6a664e2f9f 100644 --- a/salesforce/pardot.go +++ b/salesforce/pardot.go @@ -118,7 +118,8 @@ func (pc *SalesforceClientImpl) updateToken() error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - body, readErr := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + body, readErr := io.ReadAll(lmr) if readErr != nil { return fmt.Errorf("token request failed with status %d; while reading body: %w", resp.StatusCode, readErr) } @@ -126,7 +127,8 @@ func (pc *SalesforceClientImpl) updateToken() error { } var respJSON oauthTokenResp - err = json.NewDecoder(resp.Body).Decode(&respJSON) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + err = json.NewDecoder(lmr).Decode(&respJSON) if err != nil { return fmt.Errorf("failed to decode token response: %w", err) } @@ -203,7 +205,8 @@ func (pc *SalesforceClientImpl) SendContact(email string) error { return nil } - body, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + body, err := io.ReadAll(lmr) resp.Body.Close() if err != nil { diff --git a/sfe/zendesk/zendesk.go b/sfe/zendesk/zendesk.go index 46d0c5bc02a..1b18c040797 100644 --- a/sfe/zendesk/zendesk.go +++ b/sfe/zendesk/zendesk.go @@ -10,6 +10,8 @@ import ( "slices" "strings" "time" + + "github.com/letsencrypt/boulder/core" ) const ( @@ -177,7 +179,8 @@ func (c *Client) doJSONRequest(method, reqURL string, body []byte) ([]byte, erro } defer resp.Body.Close() - respBody, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 300_000}) + lmr := core.ErrOnLimitReader(resp.Body, core.DefaultMaxRead) + respBody, err := io.ReadAll(lmr) if err != nil { return nil, fmt.Errorf("failed to read zendesk response body: %w", err) }