Fix badNonce CI flake

[beautifulentropy]

Output:

Run ./tn.sh --integration
 Network boulder_default  Creating
 Network boulder_default  Created
Generating ipki/...
Generating webpki/...
echo bin/admin bin/boulder bin/ceremony bin/ct-test-srv bin/salesforce-test-srv bin/pardot-test-srv bin/chall-test-srv bin/zendesk-test-srv
bin/admin bin/boulder bin/ceremony bin/ct-test-srv bin/salesforce-test-srv bin/pardot-test-srv bin/chall-test-srv bin/zendesk-test-srv
GOBIN=/boulder/bin go install -mod=vendor -ldflags "-X \"github.com/letsencrypt/boulder/core.BuildID= +7c6ebb9c\" -X \"github.com/letsencrypt/boulder/core.BuildTime=Fri Mar  6 20:47:09 UTC 2026\" -X \"github.com/letsencrypt/boulder/core.BuildHost=root@4189cfd85fc0\"" ./...
cp bin/salesforce-test-srv bin/pardot-test-srv
2026-03-06T20:48:04.459831+00:00Z 4189cfd85fc0 unknown webpki[5054]: 6 webpki zN3Q_w [AUDIT] Process exiting normally JSON={"Command":"webpki","BuildID":"Unspecified","BuildTime":"Unspecified","GoVersion":"go1.25.5","BuildHost":"Unspecified"}
 Network boulder_publicnet2  Creating
 Network boulder_publicnet2  Created
 Network boulder_bouldernet  Creating
 Network boulder_bouldernet  Created
 Network boulder_publicnet  Creating
 Network boulder_publicnet  Created
 Container boulder-bconsul-1  Creating
 Container boulder-bjaeger-1  Creating
 Container boulder-bredis_1-1  Creating
 Container boulder-bredis_2-1  Creating
 Container boulder-bpkimetal-1  Creating
 Container boulder-bmariadb-1  Creating
 Container boulder-bvitess-1  Creating
 Container boulder-bconsul-1  Created
 Container boulder-bredis_2-1  Created
 Container boulder-bpkimetal-1  Created
 Container boulder-bvitess-1  Created
 Container boulder-bjaeger-1  Created
 Container boulder-bmariadb-1  Created
 Container boulder-bproxysql-1  Creating
 Container boulder-bredis_1-1  Created
 Container boulder-bproxysql-1  Created
 Container boulder-bpkimetal-1  Starting
 Container boulder-bvitess-1  Starting
 Container boulder-bredis_1-1  Starting
 Container boulder-bjaeger-1  Starting
 Container boulder-bmariadb-1  Starting
 Container boulder-bconsul-1  Starting
 Container boulder-bredis_2-1  Starting
 Container boulder-bconsul-1  Started
 Container boulder-bjaeger-1  Started
 Container boulder-bpkimetal-1  Started
 Container boulder-bmariadb-1  Started
 Container boulder-bproxysql-1  Starting
 Container boulder-bredis_1-1  Started
 Container boulder-bredis_2-1  Started
 Container boulder-bvitess-1  Started
 Container boulder-bproxysql-1  Started
Fri Mar  6 20:48:06 UTC 2026 - still trying to connect to boulder-mariadb:3306
Fri Mar  6 20:48:07 UTC 2026 - still trying to connect to boulder-mariadb:3306
Fri Mar  6 20:48:08 UTC 2026 - still trying to connect to boulder-mariadb:3306
Fri Mar  6 20:48:09 UTC 2026 - still trying to connect to boulder-mariadb:3306
Connected to boulder-mariadb:3306
Connected to boulder-proxysql:6033
Fri Mar  6 20:48:10 UTC 2026 - still trying to connect to boulder-vitess:33577
Fri Mar  6 20:48:11 UTC 2026 - still trying to connect to boulder-vitess:33577
Connected to boulder-vitess:33577
Connected to bpkimetal:8080

boulder_sa
Doesn't exist - creating
sql-migrate: Applied 5 migrations
Added users from ../db-users/boulder_sa.sql

incidents_sa
Doesn't exist - creating
sql-migrate: Applied 1 migration
Added users from ../db-users/incidents_sa.sql

database setup complete

Boulder Test Suite CLI

Settings:
    RUN:                integration
    BOULDER_CONFIG_DIR: test/config-next
    GOCACHE:            /boulder/.gocache/go-build-next
    UNIT_PACKAGES:      ./...
    UNIT_FLAGS:         -p=1
    FILTER:             
    COVERAGE:           false
    COVERAGE_DIR:       test/coverage/2026-03-06_20-48-12
    USE_VITESS:         false

Starting...

Running Integration Tests
echo bin/admin bin/boulder bin/ceremony bin/ct-test-srv bin/salesforce-test-srv bin/pardot-test-srv bin/chall-test-srv bin/zendesk-test-srv
bin/admin bin/boulder bin/ceremony bin/ct-test-srv bin/salesforce-test-srv bin/pardot-test-srv bin/chall-test-srv bin/zendesk-test-srv
GOBIN=/boulder/bin go install -mod=vendor  ./...
cp bin/salesforce-test-srv bin/pardot-test-srv
chall-test-srv - 2026/03/06 20:49:04 Creating HTTP-01 challenge server on 64.112.117.122:80
chall-test-srv - 2026/03/06 20:49:04 Creating HTTPS HTTP-01 challenge server on 64.112.117.122:443
chall-test-srv - 2026/03/06 20:49:04 Creating TCP and UDP DNS server on :8053
chall-test-srv - 2026/03/06 20:49:04 Creating TCP and UDP DNS server on :8054
chall-test-srv - 2026/03/06 20:49:04 Creating DoH server on :8343
chall-test-srv - 2026/03/06 20:49:04 Creating DoH server on :8443
chall-test-srv - 2026/03/06 20:49:04 Creating TLS-ALPN-01 challenge server on 64.112.117.134:443
chall-test-srv - 2026/03/06 20:49:04 Answering A queries with 64.112.117.122 by default
chall-test-srv - 2026/03/06 20:49:04 Starting challenge servers
chall-test-srv - 2026/03/06 20:49:04 Starting management server on :8055
2026/03/06 20:49:04 zendesk-test-srv listening at :9701
2026-03-06T20:49:04.682303+00:00Z deeca28ec01d unknown boulder-publisher[5057]: 6 boulder-publisher O8XOJQ Debug server listening on :8109
2026-03-06T20:49:04.682352+00:00Z deeca28ec01d unknown boulder-publisher[5057]: 6 boulder-publisher m6xNwQ [AUDIT] Process starting JSON={"Command":"boulder-publisher","BuildID":"Unspecified","BuildTime":"Unspecified","GoVersion":"go1.25.5","BuildHost":"Unspecified"}
2026-03-06T20:49:04.690402+00:00Z deeca28ec01d unknown boulder-publisher[5057]: 6 boulder-publisher 0ubX7g grpc listening on :9491

...

Starting service bad-key-revoker
All servers running. Hit ^C to kill.
    return self._post_once(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 836, in _post_once
    response = self._check_response(response, content_type=content_type)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 708, in _check_response
    raise messages.Error.from_json(jobj)
acme.messages.Error: urn:ietf:params:acme:error:badNonce :: The client sent an unacceptable anti-replay nonce :: JWS has an invalid anti-replay nonce

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/boulder/test/integration-test.py", line 158, in <module>
    main()
  File "/boulder/test/integration-test.py", line 83, in main
    run_chisel(args.test_case_filter)
  File "/boulder/test/integration-test.py", line 113, in run_chisel
    value()
  File "/boulder/test/v2_integration.py", line 137, in test_http_challenge_broken_redirect
    client = chisel2.make_client()
             ^^^^^^^^^^^^^^^^^^^^^
  File "/boulder/test/chisel2.py", line 59, in make_client
    client.net.account = client.new_account(messages.NewRegistration.from_data(email=email,
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 63, in new_account
    response = self._post(self.directory['newAccount'], new_account)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 470, in _post
    return self.net.post(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 825, in post
    return self._post_once(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 836, in _post_once
    response = self._check_response(response, content_type=content_type)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/acme/client.py", line 708, in _check_response
    raise messages.Error.from_json(jobj)
acme.messages.Error: urn:ietf:params:acme:error:badNonce :: The client sent an unacceptable anti-replay nonce :: JWS has an invalid anti-replay nonce
FAILURE while running integration

[aarongable]

Potential quick CI fix: instead of masking nonce failure with more retries, we could set maxConnectionAge to something very high.

And then we can move forward with samantha's prod fix, too.

[jsha]

In reviewing #8672, I wrote up a minimal repro for this problem that hits it 100% of the time, very quickly.

In one terminal, start up two instances of the nonce-service:

docker compose run -it --rm boulder bash
GOBIN=$PWD/bin go install ./cmd/boulder && ./bin/boulder nonce-service -config test/config/nonce-a.json -addr 10.77.77.77:9301 & ./bin/boulder nonce-service -config test/config/nonce-b.json -addr 10.77.77.77:9501 -debug-addr 10.77.77.77:9999

In another terminal, run the below newly-added nonceclient. This is a stripped-down version of the wfe's main that instantiates a nonce getter client and a nonce redeemer client, gets 10k nonces, and then redeems them all.

Note that we run these inside the boulder container so we have access to the consul SRV records.

Also note you'll have to tweak maxConnectionAge in test/config/nonce-*.json down to "1s".

Also an interesting thing of note: if I only run once instance of nonce-service (nonce-a.json), the nonceclient run succeeds! It's only when there are two instances that it fails, even if the second instance only redeems nonces that we'll never see.

GOBIN=$PWD/bin go install ./cmd/nonceclient/ && ./bin/nonceclient test/config/wfe2.json
...
nonceclient tFsUOg redeeming nonce: rpc error: code = Unavailable desc = no backends match the nonce prefix

package main

import (
	"context"
	"github.com/jmhodges/clock"
	"github.com/letsencrypt/boulder/cmd"
	wfemain "github.com/letsencrypt/boulder/cmd/boulder-wfe2"
	bgrpc "github.com/letsencrypt/boulder/grpc"
	"github.com/letsencrypt/boulder/nonce"
	noncepb "github.com/letsencrypt/boulder/nonce/proto"
	"google.golang.org/protobuf/types/known/emptypb"
	"os"
	"time"
)

func main() {
	var c wfemain.Config

	err := cmd.ReadConfigFile(os.Args[1], &c)
	cmd.FailOnError(err, "Reading JSON config file into config structure")

	stats, logger, _ := cmd.StatsAndLogging(c.Syslog, c.OpenTelemetry, c.WFE.DebugAddr)
	cmd.LogStartup(logger)

	clk := clock.New()

	tlsConfig, err := c.WFE.TLS.Load(stats)
	cmd.FailOnError(err, "TLS config")

	getNonceConn, err := bgrpc.ClientSetup(c.WFE.GetNonceService, tlsConfig, stats, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to get nonce service")
	gnc := nonce.NewGetter(getNonceConn)

	redeemNonceConn, err := bgrpc.ClientSetup(c.WFE.RedeemNonceService, tlsConfig, stats, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to redeem nonce service")
	rnc := nonce.NewRedeemer(redeemNonceConn)

	noncePrefixKey, err := c.WFE.NonceHMACKey.Load()
	cmd.FailOnError(err, "Failed to load nonceHMACKey file")

	ctx := context.Background()

	var nonces []*noncepb.NonceMessage
	for i := 0; i < 10000; i++ {
		nonceMsg, err := gnc.Nonce(ctx, &emptypb.Empty{})
		if err != nil {
			logger.Warningf("getting nonce: %v", err)
			return
		}

		nonces = append(nonces, nonceMsg)
	}

	for _, nonceMsg := range nonces {
		ctx = context.WithValue(ctx, nonce.PrefixCtxKey{}, nonceMsg.Nonce[:nonce.PrefixLen])
		ctx = context.WithValue(ctx, nonce.HMACKeyCtxKey{}, noncePrefixKey)

		_, err = rnc.Redeem(ctx, &noncepb.NonceMessage{Nonce: nonceMsg.Nonce})
		if err != nil {
			logger.Warningf("redeeming nonce: %v", err)
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
}