Skip to content

Commit bff54c5

Browse files
sandy2008claude
andauthored
fix(integration): log fuzz seed so failures can be reproduced (#7552)
Adds a newFuzzRand(t *testing.T) *rand.Rand helper that logs the random seed every fuzz test draws (via t.Logf), and replaces 13 ad-hoc rand.New(rand.NewSource(time.Now().Unix())) call sites across integration/query_fuzz_test.go (11 sites) and integration/parquet_querier_test.go (2 sites) with calls to it. A FUZZ_SEED environment variable overrides the default time-based seed so a failing CI run can be replayed locally byte-for-byte. This is a pure observability change. It does not fix the underlying TestProtobufCodecFuzz flake tracked in #7548 -- it makes the next failure classifiable. The current code silently consumes the seed via rand.New(rand.NewSource(now.Unix())) and never surfaces it in the test log, so the failing query/series combination cannot be regenerated and every failure has to be diagnosed from the comparison output alone. That is insufficient to distinguish real Cortex/Prometheus divergence from known-class non-determinism from infra noise. Logging the seed (and accepting an override) is the minimum surface needed to make the next failure reproducible. Invalid FUZZ_SEED values are logged rather than silently falling back, so a typo in CI configuration surfaces immediately instead of masquerading as a successful override. The helper is applied to all 13 fuzz call sites, not just TestProtobufCodecFuzz, because the underlying observability gap is identical for every fuzz test in the integration suite and applying the helper everywhere is strictly less code than leaving twelve other tests with the same gap. This is the final PR in a four-PR series filed against issues #7545-#7548, complementing PR #7544 (which fixes #7543). Fixes #7548 Signed-off-by: Sandy Chen <ychen@monoidtech.com> Signed-off-by: Sandy Chen <Yuxuan.Chen@morganstanley.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent e912988 commit bff54c5

2 files changed

Lines changed: 31 additions & 15 deletions

File tree

integration/parquet_querier_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package integration
55
import (
66
"context"
77
"fmt"
8-
"math/rand"
98
"path/filepath"
109
"slices"
1110
"strconv"
@@ -89,7 +88,7 @@ func TestParquetFuzz(t *testing.T) {
8988
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
9089

9190
ctx := context.Background()
92-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
91+
rnd := newFuzzRand(t)
9392
dir := filepath.Join(s.SharedDir(), "data")
9493
numSeries := 10
9594
numSamples := 60
@@ -241,7 +240,7 @@ func TestParquetProjectionPushdownFuzz(t *testing.T) {
241240
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
242241

243242
ctx := context.Background()
244-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
243+
rnd := newFuzzRand(t)
245244
dir := filepath.Join(s.SharedDir(), "data")
246245
numSeries := 20
247246
numSamples := 100

integration/query_fuzz_test.go

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func TestNativeHistogramFuzz(t *testing.T) {
121121
}
122122

123123
ctx := context.Background()
124-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
124+
rnd := newFuzzRand(t)
125125

126126
dir := filepath.Join(s.SharedDir(), "data")
127127
err = os.MkdirAll(dir, os.ModePerm)
@@ -222,7 +222,7 @@ func TestExperimentalPromQLFuncsWithPrometheus(t *testing.T) {
222222
}
223223

224224
ctx := context.Background()
225-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
225+
rnd := newFuzzRand(t)
226226

227227
dir := filepath.Join(s.SharedDir(), "data")
228228
err = os.MkdirAll(dir, os.ModePerm)
@@ -357,7 +357,7 @@ func TestDisableChunkTrimmingFuzz(t *testing.T) {
357357

358358
waitUntilReady(t, context.Background(), c1, c2, `{job="test"}`, start, now)
359359

360-
rnd := rand.New(rand.NewSource(now.Unix()))
360+
rnd := newFuzzRand(t)
361361
opts := []promqlsmith.Option{
362362
// @ modifier and offset disabled: known bug in Prometheus (e.g. predict_linear with @/offset can panic).
363363
promqlsmith.WithEnabledFunctions(enabledFunctions),
@@ -538,7 +538,7 @@ func TestExpandedPostingsCacheFuzz(t *testing.T) {
538538
}
539539
}
540540

541-
rnd := rand.New(rand.NewSource(now.Unix()))
541+
rnd := newFuzzRand(t)
542542
opts := []promqlsmith.Option{
543543
// @ modifier and offset disabled: known bug in Prometheus (e.g. predict_linear with @/offset can panic).
544544
promqlsmith.WithEnabledAggrs(enabledAggrs),
@@ -770,7 +770,7 @@ func TestVerticalShardingFuzz(t *testing.T) {
770770

771771
waitUntilReady(t, context.Background(), c1, c2, `{job="test"}`, start, end)
772772

773-
rnd := rand.New(rand.NewSource(now.Unix()))
773+
rnd := newFuzzRand(t)
774774
opts := []promqlsmith.Option{
775775
// @ modifier and offset disabled: known bug in Prometheus (e.g. predict_linear with @/offset can panic).
776776
promqlsmith.WithEnabledFunctions(enabledFunctions),
@@ -886,7 +886,7 @@ func TestProtobufCodecFuzz(t *testing.T) {
886886

887887
waitUntilReady(t, context.Background(), c1, c2, `{job="test"}`, start, end)
888888

889-
rnd := rand.New(rand.NewSource(now.Unix()))
889+
rnd := newFuzzRand(t)
890890
opts := []promqlsmith.Option{
891891
// @ modifier and offset disabled: known bug in Prometheus (e.g. predict_linear with @/offset can panic).
892892
promqlsmith.WithEnabledFunctions(enabledFunctions),
@@ -1205,7 +1205,7 @@ func TestStoreGatewayLazyExpandedPostingsSeriesFuzz(t *testing.T) {
12051205
lbls = append(lbls, labels.FromStrings(labels.MetricName, metricName, "job", "test", "series", strconv.Itoa(i%200), "status_code", statusCodes[i%5]))
12061206
}
12071207
ctx := context.Background()
1208-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
1208+
rnd := newFuzzRand(t)
12091209

12101210
dir := t.TempDir()
12111211
storage, err := e2ecortex.NewS3ClientForMinio(minio, flags["-blocks-storage.s3.bucket-name"])
@@ -1360,7 +1360,7 @@ func TestStoreGatewayLazyExpandedPostingsSeriesFuzzWithPrometheus(t *testing.T)
13601360
lbls = append(lbls, labels.FromStrings(labels.MetricName, metricName, "job", "test", "series", strconv.Itoa(i%200), "status_code", statusCodes[i%5]))
13611361
}
13621362
ctx := context.Background()
1363-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
1363+
rnd := newFuzzRand(t)
13641364

13651365
dir := filepath.Join(s.SharedDir(), "data")
13661366
err = os.MkdirAll(dir, os.ModePerm)
@@ -1593,7 +1593,7 @@ func TestBackwardCompatibilityQueryFuzz(t *testing.T) {
15931593
ctx := context.Background()
15941594
waitUntilReady(t, ctx, c1, c2, `{job="test"}`, start, end)
15951595

1596-
rnd := rand.New(rand.NewSource(now.Unix()))
1596+
rnd := newFuzzRand(t)
15971597
opts := []promqlsmith.Option{
15981598
// @ modifier and offset disabled: known bug in Prometheus (e.g. predict_linear with @/offset can panic).
15991599
promqlsmith.WithEnabledFunctions(enabledFunctions),
@@ -1665,7 +1665,7 @@ func TestPrometheusCompatibilityQueryFuzz(t *testing.T) {
16651665
}
16661666

16671667
ctx := context.Background()
1668-
rnd := rand.New(rand.NewSource(time.Now().Unix()))
1668+
rnd := newFuzzRand(t)
16691669

16701670
dir := filepath.Join(s.SharedDir(), "data")
16711671
err = os.MkdirAll(dir, os.ModePerm)
@@ -1819,8 +1819,7 @@ func TestRW1vsRW2QueryFuzz(t *testing.T) {
18191819
_, err = c2.PushV2(symbols, v2Series)
18201820
require.NoError(t, err)
18211821

1822-
seed := now.Unix()
1823-
rnd := rand.New(rand.NewSource(seed))
1822+
rnd := newFuzzRand(t)
18241823

18251824
ctx := context.Background()
18261825
waitUntilReady(t, ctx, c1, c2, `{job="test"}`, start, end)
@@ -1983,6 +1982,24 @@ func shouldUseSampleNumComparer(query string) bool {
19831982
return false
19841983
}
19851984

1985+
// newFuzzRand returns a *rand.Rand whose seed is logged via t.Logf so failing
1986+
// fuzz cases can be reproduced. By default the seed is time.Now().Unix();
1987+
// setting FUZZ_SEED to a base-10 int64 overrides the default and pins the
1988+
// run to a specific seed.
1989+
func newFuzzRand(t *testing.T) *rand.Rand {
1990+
seed := time.Now().Unix()
1991+
if v := os.Getenv("FUZZ_SEED"); v != "" {
1992+
if parsed, err := strconv.ParseInt(v, 10, 64); err == nil {
1993+
t.Logf("integration fuzz random seed: overridden to %d via FUZZ_SEED", parsed)
1994+
seed = parsed
1995+
} else {
1996+
t.Logf("integration fuzz random seed: ignoring invalid FUZZ_SEED=%q: %v", v, err)
1997+
}
1998+
}
1999+
t.Logf("integration fuzz random seed: %d (override with FUZZ_SEED env var)", seed)
2000+
return rand.New(rand.NewSource(seed))
2001+
}
2002+
19862003
func isValidQuery(generatedQuery parser.Expr, skipBackwardIncompat bool) bool {
19872004
isValid := true
19882005
queryStr := generatedQuery.String()

0 commit comments

Comments
 (0)