Skip to content

Commit d1e5e16

Browse files
jprenkenjprenken
authored
wfe/ra: Periodically load rate limit overrides from the database (#8407)
Add a `refreshOverrides` func to the `ratelimits.limitRegistry` struct. Instead of populating the static `overrides` field once when creating an instance of the struct, call the new func at startup and then every 30 minutes. Emit relevant logs and metrics from `limitRegistry`. Add an `OverridesFromDB` limiter config flag (for RA & WFE) to read overrides from the DB instead of a file. Flatten `newLimitRegistry.*()` methods' logic into their sole caller, `NewTransactionBuilder()`. Rename `loadDefaults()` & `loadOverrides()`, appending `FromFile` for clarity/consistency. test: Add ra-sct-provider dependency on SA. **Important for deployment:** If the `OverridesFromDB` config flag is enabled, an RA now depends on the SA in order to load overrides. The RA must be added as a gRPC client of `sa.StorageAuthorityReadOnly`. *CPS Compliance Review:* `OverridesFromDB` only controls how we load rate limit overrides, which has no compliance implications beyond general API availability (e.g. for revocation). I've checked our CP/CPS to confirm we make no related stipulations. Fixes #8382
1 parent d924a2a commit d1e5e16

16 files changed

Lines changed: 680 additions & 124 deletions

File tree

‎cmd/boulder-ra/main.go‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"flag"
66
"os"
7+
"time"
78

89
"github.com/jmhodges/clock"
910

@@ -69,13 +70,17 @@ type Config struct {
6970

7071
// Overrides is a path to a YAML file containing overrides for the
7172
// default rate limits. See: ratelimits/README.md for details. If
72-
// this field is not set, all requesters will be subject to the
73-
// default rate limits. Overrides passed in this file must be
74-
// identical to those in the WFE.
73+
// neither this field nor OverridesFromDB is set, all requesters
74+
// will be subject to the default rate limits. Overrides passed in
75+
// this file must be identical to those in the WFE.
7576
//
7677
// Note: At this time, only the Failed Authorizations overrides are
7778
// necessary in the RA.
7879
Overrides string
80+
81+
// OverridesFromDB causes the WFE and RA to retrieve rate limit overrides
82+
// from the database, instead of from a file.
83+
OverridesFromDB bool
7984
}
8085

8186
// MaxNames is the maximum number of subjectAltNames in a single cert.
@@ -271,8 +276,18 @@ func main() {
271276
source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, scope)
272277
limiter, err = ratelimits.NewLimiter(clk, source, scope)
273278
cmd.FailOnError(err, "Failed to create rate limiter")
274-
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides)
279+
if c.RA.Limiter.OverridesFromDB {
280+
if c.RA.Limiter.Overrides != "" {
281+
cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
282+
}
283+
saroc := sapb.NewStorageAuthorityReadOnlyClient(saConn)
284+
txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.RA.Limiter.Defaults, saroc.GetEnabledRateLimitOverrides, scope, logger)
285+
} else {
286+
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides, scope, logger)
287+
}
275288
cmd.FailOnError(err, "Failed to create rate limits transaction builder")
289+
overrideRefresherShutdown := txnBuilder.NewRefresher(30 * time.Minute)
290+
defer overrideRefresherShutdown()
276291
}
277292

278293
rai := ra.NewRegistrationAuthorityImpl(

‎cmd/boulder-wfe2/main.go‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,15 @@ type Config struct {
151151

152152
// Overrides is a path to a YAML file containing overrides for the
153153
// default rate limits. See: ratelimits/README.md for details. If
154-
// this field is not set, all requesters will be subject to the
155-
// default rate limits. Overrides for the Failed Authorizations
156-
// overrides passed in this file must be identical to those in the
157-
// RA.
154+
// neither this field nor OverridesFromDB is set, all requesters
155+
// will be subject to the default rate limits. Overrides for the
156+
// Failed Authorizations overrides passed in this file must be
157+
// identical to those in the RA.
158158
Overrides string
159+
160+
// OverridesFromDB causes the WFE and RA to retrieve rate limit
161+
// overrides from the database, instead of from a file.
162+
OverridesFromDB bool
159163
}
160164

161165
// CertProfiles is a map of acceptable certificate profile names to
@@ -326,6 +330,7 @@ func main() {
326330
var limiter *ratelimits.Limiter
327331
var txnBuilder *ratelimits.TransactionBuilder
328332
var limiterRedis *bredis.Ring
333+
overridesRefresherShutdown := func() {}
329334
if c.WFE.Limiter.Defaults != "" {
330335
// Setup rate limiting.
331336
limiterRedis, err = bredis.NewRingFromConfig(*c.WFE.Limiter.Redis, stats, logger)
@@ -334,8 +339,16 @@ func main() {
334339
source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
335340
limiter, err = ratelimits.NewLimiter(clk, source, stats)
336341
cmd.FailOnError(err, "Failed to create rate limiter")
337-
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides)
342+
if c.WFE.Limiter.OverridesFromDB {
343+
if c.WFE.Limiter.Overrides != "" {
344+
cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
345+
}
346+
txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.WFE.Limiter.Defaults, sac.GetEnabledRateLimitOverrides, stats, logger)
347+
} else {
348+
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides, stats, logger)
349+
}
338350
cmd.FailOnError(err, "Failed to create rate limits transaction builder")
351+
overridesRefresherShutdown = txnBuilder.NewRefresher(30 * time.Minute)
339352
}
340353

341354
var accountGetter wfe2.AccountGetter
@@ -413,6 +426,7 @@ func main() {
413426
defer func() {
414427
ctx, cancel := context.WithTimeout(context.Background(), c.WFE.ShutdownStopTimeout.Duration)
415428
defer cancel()
429+
overridesRefresherShutdown()
416430
_ = srv.Shutdown(ctx)
417431
_ = tlsSrv.Shutdown(ctx)
418432
limiterRedis.StopLookups()

‎cmd/sfe/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ func main() {
223223
source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
224224
limiter, err = ratelimits.NewLimiter(clk, source, stats)
225225
cmd.FailOnError(err, "Failed to create rate limiter")
226-
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "")
226+
txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "", stats, logger)
227227
cmd.FailOnError(err, "Failed to create rate limits transaction builder")
228228
}
229229

‎ra/ra.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ type RegistrationAuthorityImpl struct {
8585
maxContactsPerReg int
8686
limiter *ratelimits.Limiter
8787
txnBuilder *ratelimits.TransactionBuilder
88+
started time.Time
8889
finalizeTimeout time.Duration
8990
drainWG sync.WaitGroup
9091

@@ -110,6 +111,15 @@ type RegistrationAuthorityImpl struct {
110111

111112
var _ rapb.RegistrationAuthorityServer = (*RegistrationAuthorityImpl)(nil)
112113

114+
// Health implements our grpc.checker interface. This method will be called
115+
// periodically to set the gRPC service's healthpb.Health.Check() status.
116+
func (ra *RegistrationAuthorityImpl) Health(ctx context.Context) error {
117+
if ra.txnBuilder.Ready() || time.Since(ra.started) > time.Second*10 {
118+
return nil
119+
}
120+
return errors.New("waiting for overrides")
121+
}
122+
113123
// NewRegistrationAuthorityImpl constructs a new RA object.
114124
func NewRegistrationAuthorityImpl(
115125
clk clock.Clock,
@@ -234,6 +244,7 @@ func NewRegistrationAuthorityImpl(
234244
keyPolicy: keyPolicy,
235245
limiter: limiter,
236246
txnBuilder: txnBuilder,
247+
started: clk.Now(),
237248
publisher: pubc,
238249
finalizeTimeout: finalizeTimeout,
239250
ctpolicy: ctp,

‎ra/ra_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, sapb.StorageAutho
363363
rlSource := ratelimits.NewInmemSource()
364364
limiter, err := ratelimits.NewLimiter(fc, rlSource, stats)
365365
test.AssertNotError(t, err, "making limiter")
366-
txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "")
366+
txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "", metrics.NoopRegisterer, log)
367367
test.AssertNotError(t, err, "making transaction composer")
368368

369369
testKeyPolicy, err := goodkey.NewPolicy(nil, nil)
@@ -708,7 +708,7 @@ func TestPerformValidation_FailedValidationsTriggerPauseIdentifiersRatelimit(t *
708708
Burst: 1,
709709
Count: 1,
710710
Period: config.Duration{Duration: time.Hour * 24}},
711-
})
711+
}, nil, metrics.NoopRegisterer, blog.NewMock())
712712
test.AssertNotError(t, err, "making transaction composer")
713713
ra.txnBuilder = txnBuilder
714714

@@ -967,7 +967,7 @@ func TestDeactivateAuthorization_Pausing(t *testing.T) {
967967
Burst: 1,
968968
Count: 1,
969969
Period: config.Duration{Duration: time.Hour * 24}},
970-
})
970+
}, nil, metrics.NoopRegisterer, blog.NewMock())
971971
test.AssertNotError(t, err, "making transaction composer")
972972
ra.txnBuilder = txnBuilder
973973

0 commit comments

Comments
 (0)