From 5ada8697a7d9a0f077a5b9baad942f4b147c4559 Mon Sep 17 00:00:00 2001 From: Samantha Date: Wed, 25 Mar 2026 14:09:35 -0400 Subject: [PATCH 1/2] va: Add experimental VA for testing Hickory --- cmd/boulder-va/main.go | 69 +++++++++++++++++++++++- cmd/remoteva/main.go | 2 + va/caa.go | 20 ++++++- va/va.go | 116 +++++++++++++++++++++++++++++++--------- va/va_test.go | 117 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 298 insertions(+), 26 deletions(-) diff --git a/cmd/boulder-va/main.go b/cmd/boulder-va/main.go index 594221dab0c..66bc7888075 100644 --- a/cmd/boulder-va/main.go +++ b/cmd/boulder-va/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/jmhodges/clock" + "github.com/prometheus/client_golang/prometheus" "github.com/letsencrypt/boulder/bdns" "github.com/letsencrypt/boulder/cmd" @@ -57,7 +58,23 @@ type Config struct { // when the VA first gets a quorum of (un)successful remote results. // Leaving this value zero means the VA won't early-cancel slow remotes. SlowRemoteTimeout config.Duration - Features features.Config + + // ExperimentalVA configures an optional parallel VA that shadows the + // primary VA's DCV and CAA checks using an alternative DNS resolver, + // emitting comparison metrics without affecting the real validation + // decision. + ExperimentalVA *struct { + // DNSProvider is the dynamic DNS provider config for the + // experimental VA's resolver. + DNSProvider *cmd.DNSProvider `validate:"required"` + // DNSTimeout is the timeout for DNS queries. Defaults to the + // primary VA's DNSTimeout if unset. + DNSTimeout config.Duration `validate:"omitempty"` + // SampleRate controls the rate of validations that are shadowed + // (0.0 to 1.0). A value of 0 disables shadowing. + SampleRate float64 `validate:"min=0,max=1"` + } + Features features.Config } Syslog cmd.SyslogConfig @@ -130,6 +147,54 @@ func main() { } } + var experimentalVA *va.ValidationAuthorityImpl + var experimentalVASampleRate float64 + if c.VA.ExperimentalVA != nil { + servers, err := bdns.StartDynamicProvider(c.VA.ExperimentalVA.DNSProvider, 60*time.Second, "tcp") + cmd.FailOnError(err, "Couldn't start experimental dynamic DNS server resolver") + defer servers.Stop() + + dnsTimeout := c.VA.ExperimentalVA.DNSTimeout.Duration + if dnsTimeout <= 0 { + dnsTimeout = c.VA.DNSTimeout.Duration + } + + // Prefix experimental VA metrics to avoid metric name collisions with + // the primary VA. + scope := prometheus.WrapRegistererWithPrefix("experimental_", scope) + + resolver := bdns.New( + dnsTimeout, + servers, + scope, + clk, + c.VA.DNSTries, + c.VA.UserAgent, + logger, + tlsConfig, + ) + + experimentalVA, err = va.NewValidationAuthorityImpl( + resolver, + nil, + c.VA.UserAgent, + c.VA.IssuerDomain, + scope, + clk, + logger, + c.VA.AccountURIPrefixes, + "Experimental", + "", + iana.IsReservedAddr, + 0, + c.VA.DNSAllowLoopbackAddresses, + nil, + 0, + ) + cmd.FailOnError(err, "Unable to create experimental VA") + experimentalVASampleRate = c.VA.ExperimentalVA.SampleRate + } + vai, err := va.NewValidationAuthorityImpl( resolver, remotes, @@ -144,6 +209,8 @@ func main() { iana.IsReservedAddr, c.VA.SlowRemoteTimeout.Duration, c.VA.DNSAllowLoopbackAddresses, + experimentalVA, + experimentalVASampleRate, ) cmd.FailOnError(err, "Unable to create VA server") diff --git a/cmd/remoteva/main.go b/cmd/remoteva/main.go index 7309146427e..47d83f94427 100644 --- a/cmd/remoteva/main.go +++ b/cmd/remoteva/main.go @@ -130,6 +130,8 @@ func main() { iana.IsReservedAddr, 0, c.RVA.DNSAllowLoopbackAddresses, + nil, + 0, ) cmd.FailOnError(err, "Unable to create Remote-VA server") diff --git a/va/caa.go b/va/caa.go index c64a8333f32..7869a40d904 100644 --- a/va/caa.go +++ b/va/caa.go @@ -103,7 +103,25 @@ func (va *ValidationAuthorityImpl) DoCAA(ctx context.Context, req *vapb.IsCAAVal logEvent.InternalError = err.Error() prob = detailedError(err) prob.Detail = fmt.Sprintf("While processing CAA for %s: %s", ident.Value, prob.Detail) - return bgrpc.CAAResultToPB(filterProblemDetails(prob), va.perspective, va.rir) + } + + var localResult remoteResult + if va.shouldDispatchExperiment() { + defer func() { + va.dispatchExperiment(opCAA, localResult, func(ctx context.Context) (remoteResult, error) { + return va.experimentalVA.DoCAA(ctx, req) + }) + }() + } + + // Capture the local validation result for experimental resolver comparison + // before MPIC can influence the outcome. + localResult, err = bgrpc.CAAResultToPB(filterProblemDetails(prob), va.perspective, va.rir) + if err != nil { + return nil, err + } + if prob != nil { + return localResult.(*vapb.IsCAAValidResponse), nil } if va.isPrimaryVA() { diff --git a/va/va.go b/va/va.go index efdf30e83ff..97cee98513a 100644 --- a/va/va.go +++ b/va/va.go @@ -113,6 +113,11 @@ type vaMetrics struct { http01Redirects prometheus.Counter caaCounter *prometheus.CounterVec ipv4FallbackCounter prometheus.Counter + // experimentConcurrence tracks whether the primary and experimental VAs + // reached the same outcome. It's labelled by: + // - operation: [dcv|caa] + // - concurrence: [true|false] + experimentConcurrence *prometheus.CounterVec } func initMetrics(stats prometheus.Registerer) *vaMetrics { @@ -145,6 +150,10 @@ func initMetrics(stats prometheus.Registerer) *vaMetrics { Name: "tls_alpn_ipv4_fallback", Help: "A counter of IPv4 fallbacks during TLS ALPN validation", }) + experimentConcurrence := promauto.With(stats).NewCounterVec(prometheus.CounterOpts{ + Name: "experiment_concurrence", + Help: "Count of validations where the experimental VA did or did not concur with the primary VA", + }, []string{"operation", "concurrence"}) return &vaMetrics{ validationLatency: validationLatency, @@ -154,6 +163,7 @@ func initMetrics(stats prometheus.Registerer) *vaMetrics { http01Redirects: http01Redirects, caaCounter: caaCounter, ipv4FallbackCounter: ipv4FallbackCounter, + experimentConcurrence: experimentConcurrence, } } @@ -188,23 +198,25 @@ func newDefaultPortConfig() *portConfig { type ValidationAuthorityImpl struct { vapb.UnsafeVAServer vapb.UnsafeCAAServer - log blog.Logger - dnsClient bdns.Client - issuerDomain string - httpPort int - httpsPort int - tlsPort int - userAgent string - clk clock.Clock - remoteVAs []RemoteVA - maxRemoteFailures int - accountURIPrefixes []string - singleDialTimeout time.Duration - slowRemoteTimeout time.Duration - perspective string - rir string - isReservedIPFunc func(netip.Addr) error - allowRestrictedAddrs bool + log blog.Logger + dnsClient bdns.Client + issuerDomain string + httpPort int + httpsPort int + tlsPort int + userAgent string + clk clock.Clock + remoteVAs []RemoteVA + maxRemoteFailures int + accountURIPrefixes []string + singleDialTimeout time.Duration + slowRemoteTimeout time.Duration + perspective string + rir string + isReservedIPFunc func(netip.Addr) error + allowRestrictedAddrs bool + experimentalVA *ValidationAuthorityImpl + experimentalVASampleRate float64 metrics *vaMetrics } @@ -227,6 +239,8 @@ func NewValidationAuthorityImpl( reservedIPChecker func(netip.Addr) error, slowRemoteTimeout time.Duration, allowRestrictedAddrs bool, + experimentalVA *ValidationAuthorityImpl, + experimentalVASampleRate float64, ) (*ValidationAuthorityImpl, error) { if len(accountURIPrefixes) == 0 { @@ -268,17 +282,53 @@ func NewValidationAuthorityImpl( // before timing out. This timeout ignores the base RPC timeout and is strictly // used for the DialContext operations that take place during an // HTTP-01 challenge validation. - singleDialTimeout: 10 * time.Second, - slowRemoteTimeout: slowRemoteTimeout, - perspective: perspective, - rir: rir, - isReservedIPFunc: reservedIPChecker, - allowRestrictedAddrs: allowRestrictedAddrs, + singleDialTimeout: 10 * time.Second, + slowRemoteTimeout: slowRemoteTimeout, + perspective: perspective, + rir: rir, + isReservedIPFunc: reservedIPChecker, + allowRestrictedAddrs: allowRestrictedAddrs, + experimentalVA: experimentalVA, + experimentalVASampleRate: experimentalVASampleRate, } return va, nil } +func (va *ValidationAuthorityImpl) shouldDispatchExperiment() bool { + return va.experimentalVA != nil && rand.Float64() < va.experimentalVASampleRate +} + +func (va *ValidationAuthorityImpl) dispatchExperiment(operation string, primary remoteResult, experimentFunc func(context.Context) (remoteResult, error)) { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), va.slowRemoteTimeout) + defer cancel() + + experimentResult, err := experimentFunc(ctx) + + primaryPassed := primary.GetProblem() == nil + experimentPassed := (err == nil) && (experimentResult.GetProblem() == nil) + + if primaryPassed == experimentPassed { + va.metrics.experimentConcurrence.WithLabelValues(operation, "true").Inc() + return + } + va.metrics.experimentConcurrence.WithLabelValues(operation, "false").Inc() + + logArgs := map[string]any{ + "operation": operation, + "primaryPassed": primaryPassed, + "primaryResult": primary, + "experimentPassed": experimentPassed, + "experimentResult": experimentResult, + } + if err != nil { + logArgs["experimentErr"] = err.Error() + } + va.log.AuditInfo("Primary VA disagreed with experimental VA", logArgs) + }() +} + // maxAllowedFailures returns the maximum number of allowed failures // for a given number of remote perspectives, according to the "Quorum // Requirements" table in BRs Section 3.2.2.9, as follows: @@ -767,7 +817,25 @@ func (va *ValidationAuthorityImpl) DoDCV(ctx context.Context, req *vapb.PerformV if err != nil { logEvent.InternalError = err.Error() prob = detailedError(err) - return bgrpc.ValidationResultToPB(records, filterProblemDetails(prob), va.perspective, va.rir) + } + + var localResult remoteResult + if va.shouldDispatchExperiment() { + defer func() { + va.dispatchExperiment(opDCV, localResult, func(ctx context.Context) (remoteResult, error) { + return va.experimentalVA.DoDCV(ctx, req) + }) + }() + } + + // Capture the local validation result for experimental resolver comparison + // before MPIC can influence the outcome. + localResult, err = bgrpc.ValidationResultToPB(records, filterProblemDetails(prob), va.perspective, va.rir) + if err != nil { + return nil, err + } + if prob != nil { + return localResult.(*vapb.ValidationResult), nil } if va.isPrimaryVA() { diff --git a/va/va_test.go b/va/va_test.go index e6f1d77e151..39991b29bca 100644 --- a/va/va_test.go +++ b/va/va_test.go @@ -152,6 +152,8 @@ func setup(srv *httptest.Server, userAgent string, remoteVAs []RemoteVA, fakeDNS isNonLoopbackReservedIP, time.Second, true, + nil, + 0, ) if err != nil { panic(fmt.Sprintf("Failed to create validation authority: %v", err)) @@ -328,6 +330,8 @@ func TestNewValidationAuthorityImplWithDuplicateRemotes(t *testing.T) { isNonLoopbackReservedIP, time.Second, true, + nil, + 0, ) test.AssertError(t, err, "NewValidationAuthorityImpl allowed duplicate remote perspectives") test.AssertContains(t, err.Error(), "duplicate remote VA perspective \"dadaist\"") @@ -388,6 +392,119 @@ func TestValidateMalformedChallenge(t *testing.T) { test.AssertEquals(t, prob.Type, probs.MalformedProblem) } +func setupWithExperimental(experimentalDNS bdns.Client) (*ValidationAuthorityImpl, *blog.Mock) { + va, mockLog := setup(nil, "", nil, &txtFakeDNS{}) + expVA, _ := setup(nil, "", nil, experimentalDNS) + expVA.perspective = "Experimental" + va.experimentalVA = expVA + va.experimentalVASampleRate = 1.0 + return va, mockLog +} + +func TestExperimentalVAConcurrence(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + experimentalDNS bdns.Client + domain string + primaryProblem bool + expectConcur float64 + expectDisagree float64 + expectLog string + }{ + { + name: "both pass", + experimentalDNS: &txtFakeDNS{}, + domain: "good-dns01.com", + expectConcur: 1, + expectDisagree: 0, + }, + { + name: "primary passes experimental fails", + experimentalDNS: &bdns.MockClient{}, + domain: "good-dns01.com", + expectConcur: 0, + expectDisagree: 1, + expectLog: "Primary VA disagreed with experimental VA", + }, + { + name: "both fail", + experimentalDNS: &bdns.MockClient{}, + domain: "servfail.com", + primaryProblem: true, + expectConcur: 1, + expectDisagree: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + va, mockLog := setupWithExperimental(tc.experimentalDNS) + req := createValidationRequest(identifier.NewDNS(tc.domain), core.ChallengeTypeDNS01) + res, err := va.DoDCV(context.Background(), req) + test.AssertNotError(t, err, "DoDCV failed") + if tc.primaryProblem { + test.Assert(t, res.Problem != nil, "expected a problem") + } else { + test.Assert(t, res.Problem == nil, "expected no problem") + } + + time.Sleep(100 * time.Millisecond) + + test.AssertMetricWithLabelsEquals(t, va.metrics.experimentConcurrence, prometheus.Labels{ + "operation": opDCV, + "concurrence": "true", + }, tc.expectConcur) + test.AssertMetricWithLabelsEquals(t, va.metrics.experimentConcurrence, prometheus.Labels{ + "operation": opDCV, + "concurrence": "false", + }, tc.expectDisagree) + + if tc.expectLog != "" { + test.AssertEquals(t, len(mockLog.GetAllMatching(tc.expectLog)), 1) + } + }) + } +} + +func TestExperimentalVANilIsNoop(t *testing.T) { + t.Parallel() + + va, _ := setup(nil, "", nil, &txtFakeDNS{}) + req := createValidationRequest(identifier.NewDNS("good-dns01.com"), core.ChallengeTypeDNS01) + res, err := va.DoDCV(context.Background(), req) + test.AssertNotError(t, err, "DoDCV failed") + test.Assert(t, res.Problem == nil, "validation should succeed") +} + +func TestExperimentalVAZeroSample(t *testing.T) { + t.Parallel() + + va, _ := setup(nil, "", nil, &txtFakeDNS{}) + expVA, _ := setup(nil, "", nil, &bdns.MockClient{}) + expVA.perspective = "Experimental" + va.experimentalVA = expVA + va.experimentalVASampleRate = 0 + + req := createValidationRequest(identifier.NewDNS("good-dns01.com"), core.ChallengeTypeDNS01) + res, err := va.DoDCV(context.Background(), req) + test.AssertNotError(t, err, "DoDCV failed") + test.Assert(t, res.Problem == nil, "expected no problem") + + time.Sleep(100 * time.Millisecond) + + test.AssertMetricWithLabelsEquals(t, va.metrics.experimentConcurrence, prometheus.Labels{ + "operation": opDCV, + "concurrence": "true", + }, 0) + test.AssertMetricWithLabelsEquals(t, va.metrics.experimentConcurrence, prometheus.Labels{ + "operation": opDCV, + "concurrence": "false", + }, 0) +} + func TestPerformValidationInvalid(t *testing.T) { t.Parallel() va, _ := setup(nil, "", nil, &txtFakeDNS{}) From c9f3e5aa4c59146a70f6454e51775c93b6c02611 Mon Sep 17 00:00:00 2001 From: Samantha Date: Thu, 26 Mar 2026 15:19:03 -0400 Subject: [PATCH 2/2] Address comments --- cmd/boulder-va/main.go | 14 ++++++-- cmd/remoteva/main.go | 1 + va/caa.go | 24 +++++++------ va/va.go | 81 +++++++++++++++++++++++------------------- va/va_test.go | 2 ++ 5 files changed, 71 insertions(+), 51 deletions(-) diff --git a/cmd/boulder-va/main.go b/cmd/boulder-va/main.go index 66bc7888075..d5b48097f68 100644 --- a/cmd/boulder-va/main.go +++ b/cmd/boulder-va/main.go @@ -59,7 +59,7 @@ type Config struct { // Leaving this value zero means the VA won't early-cancel slow remotes. SlowRemoteTimeout config.Duration - // ExperimentalVA configures an optional parallel VA that shadows the + // ExperimentalVA configures an optional parallel VA that repeats the // primary VA's DCV and CAA checks using an alternative DNS resolver, // emitting comparison metrics without affecting the real validation // decision. @@ -70,9 +70,13 @@ type Config struct { // DNSTimeout is the timeout for DNS queries. Defaults to the // primary VA's DNSTimeout if unset. DNSTimeout config.Duration `validate:"omitempty"` - // SampleRate controls the rate of validations that are shadowed - // (0.0 to 1.0). A value of 0 disables shadowing. + // SampleRate controls the rate of validations that are repeated + // (0.0 to 1.0). A value of 0 disables it entirely, while 1 repeats + // all validations. SampleRate float64 `validate:"min=0,max=1"` + // Timeout is the timeout for experimental validation operations. + // This should be configured to match the RA->VA timeout. + Timeout config.Duration `validate:"required"` } Features features.Config } @@ -149,6 +153,7 @@ func main() { var experimentalVA *va.ValidationAuthorityImpl var experimentalVASampleRate float64 + var experimentalVATimeout time.Duration if c.VA.ExperimentalVA != nil { servers, err := bdns.StartDynamicProvider(c.VA.ExperimentalVA.DNSProvider, 60*time.Second, "tcp") cmd.FailOnError(err, "Couldn't start experimental dynamic DNS server resolver") @@ -190,9 +195,11 @@ func main() { c.VA.DNSAllowLoopbackAddresses, nil, 0, + 0, ) cmd.FailOnError(err, "Unable to create experimental VA") experimentalVASampleRate = c.VA.ExperimentalVA.SampleRate + experimentalVATimeout = c.VA.ExperimentalVA.Timeout.Duration } vai, err := va.NewValidationAuthorityImpl( @@ -211,6 +218,7 @@ func main() { c.VA.DNSAllowLoopbackAddresses, experimentalVA, experimentalVASampleRate, + experimentalVATimeout, ) cmd.FailOnError(err, "Unable to create VA server") diff --git a/cmd/remoteva/main.go b/cmd/remoteva/main.go index 47d83f94427..8fd9da293ac 100644 --- a/cmd/remoteva/main.go +++ b/cmd/remoteva/main.go @@ -132,6 +132,7 @@ func main() { c.RVA.DNSAllowLoopbackAddresses, nil, 0, + 0, ) cmd.FailOnError(err, "Unable to create Remote-VA server") diff --git a/va/caa.go b/va/caa.go index 7869a40d904..95166cb08ae 100644 --- a/va/caa.go +++ b/va/caa.go @@ -105,23 +105,25 @@ func (va *ValidationAuthorityImpl) DoCAA(ctx context.Context, req *vapb.IsCAAVal prob.Detail = fmt.Sprintf("While processing CAA for %s: %s", ident.Value, prob.Detail) } - var localResult remoteResult - if va.shouldDispatchExperiment() { - defer func() { - va.dispatchExperiment(opCAA, localResult, func(ctx context.Context) (remoteResult, error) { - return va.experimentalVA.DoCAA(ctx, req) - }) - }() - } - // Capture the local validation result for experimental resolver comparison // before MPIC can influence the outcome. - localResult, err = bgrpc.CAAResultToPB(filterProblemDetails(prob), va.perspective, va.rir) + localResult, err := bgrpc.CAAResultToPB(filterProblemDetails(prob), va.perspective, va.rir) if err != nil { return nil, err } + + if va.shouldRunExperiment() { + go va.runExperiment( + ctx, + opCAA, + proto.Clone(localResult).(*vapb.IsCAAValidResponse), + func(ctx context.Context) (remoteResult, error) { + return va.experimentalVA.DoCAA(ctx, req) + }) + } + if prob != nil { - return localResult.(*vapb.IsCAAValidResponse), nil + return localResult, nil } if va.isPrimaryVA() { diff --git a/va/va.go b/va/va.go index 97cee98513a..88f13dc5ef8 100644 --- a/va/va.go +++ b/va/va.go @@ -217,6 +217,7 @@ type ValidationAuthorityImpl struct { allowRestrictedAddrs bool experimentalVA *ValidationAuthorityImpl experimentalVASampleRate float64 + experimentalVATimeout time.Duration metrics *vaMetrics } @@ -241,6 +242,7 @@ func NewValidationAuthorityImpl( allowRestrictedAddrs bool, experimentalVA *ValidationAuthorityImpl, experimentalVASampleRate float64, + experimentalVATimeout time.Duration, ) (*ValidationAuthorityImpl, error) { if len(accountURIPrefixes) == 0 { @@ -290,43 +292,46 @@ func NewValidationAuthorityImpl( allowRestrictedAddrs: allowRestrictedAddrs, experimentalVA: experimentalVA, experimentalVASampleRate: experimentalVASampleRate, + experimentalVATimeout: experimentalVATimeout, } return va, nil } -func (va *ValidationAuthorityImpl) shouldDispatchExperiment() bool { +func (va *ValidationAuthorityImpl) shouldRunExperiment() bool { return va.experimentalVA != nil && rand.Float64() < va.experimentalVASampleRate } -func (va *ValidationAuthorityImpl) dispatchExperiment(operation string, primary remoteResult, experimentFunc func(context.Context) (remoteResult, error)) { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), va.slowRemoteTimeout) - defer cancel() +// runExperiment compares the primary VA's local result against the experimental +// VA's result and records a concurrence metric. On disagreement, it logs a +// structured event with both results. The primary argument must be non-nil. +// Callers should invoke this in a goroutine. +func (va *ValidationAuthorityImpl) runExperiment(ctx context.Context, operation string, primary remoteResult, experimentFunc func(context.Context) (remoteResult, error)) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), va.experimentalVATimeout) + defer cancel() - experimentResult, err := experimentFunc(ctx) + experimentResult, err := experimentFunc(ctx) - primaryPassed := primary.GetProblem() == nil - experimentPassed := (err == nil) && (experimentResult.GetProblem() == nil) + primaryPassed := primary.GetProblem() == nil + experimentPassed := (err == nil) && (experimentResult.GetProblem() == nil) - if primaryPassed == experimentPassed { - va.metrics.experimentConcurrence.WithLabelValues(operation, "true").Inc() - return - } - va.metrics.experimentConcurrence.WithLabelValues(operation, "false").Inc() - - logArgs := map[string]any{ - "operation": operation, - "primaryPassed": primaryPassed, - "primaryResult": primary, - "experimentPassed": experimentPassed, - "experimentResult": experimentResult, - } - if err != nil { - logArgs["experimentErr"] = err.Error() - } - va.log.AuditInfo("Primary VA disagreed with experimental VA", logArgs) - }() + if primaryPassed == experimentPassed { + va.metrics.experimentConcurrence.WithLabelValues(operation, "true").Inc() + return + } + va.metrics.experimentConcurrence.WithLabelValues(operation, "false").Inc() + + logArgs := map[string]any{ + "operation": operation, + "primaryPassed": primaryPassed, + "primaryResult": primary, + "experimentPassed": experimentPassed, + "experimentResult": experimentResult, + } + if err != nil { + logArgs["experimentErr"] = err.Error() + } + va.log.AuditInfo("Primary VA disagreed with experimental VA", logArgs) } // maxAllowedFailures returns the maximum number of allowed failures @@ -819,23 +824,25 @@ func (va *ValidationAuthorityImpl) DoDCV(ctx context.Context, req *vapb.PerformV prob = detailedError(err) } - var localResult remoteResult - if va.shouldDispatchExperiment() { - defer func() { - va.dispatchExperiment(opDCV, localResult, func(ctx context.Context) (remoteResult, error) { - return va.experimentalVA.DoDCV(ctx, req) - }) - }() - } - // Capture the local validation result for experimental resolver comparison // before MPIC can influence the outcome. - localResult, err = bgrpc.ValidationResultToPB(records, filterProblemDetails(prob), va.perspective, va.rir) + localResult, err := bgrpc.ValidationResultToPB(records, filterProblemDetails(prob), va.perspective, va.rir) if err != nil { return nil, err } + + if va.shouldRunExperiment() { + go va.runExperiment( + ctx, + opDCV, + proto.Clone(localResult).(*vapb.ValidationResult), + func(ctx context.Context) (remoteResult, error) { + return va.experimentalVA.DoDCV(ctx, req) + }) + } + if prob != nil { - return localResult.(*vapb.ValidationResult), nil + return localResult, nil } if va.isPrimaryVA() { diff --git a/va/va_test.go b/va/va_test.go index 39991b29bca..2a9f2e88ac7 100644 --- a/va/va_test.go +++ b/va/va_test.go @@ -154,6 +154,7 @@ func setup(srv *httptest.Server, userAgent string, remoteVAs []RemoteVA, fakeDNS true, nil, 0, + 0, ) if err != nil { panic(fmt.Sprintf("Failed to create validation authority: %v", err)) @@ -332,6 +333,7 @@ func TestNewValidationAuthorityImplWithDuplicateRemotes(t *testing.T) { true, nil, 0, + 0, ) test.AssertError(t, err, "NewValidationAuthorityImpl allowed duplicate remote perspectives") test.AssertContains(t, err.Error(), "duplicate remote VA perspective \"dadaist\"")