diff --git a/common/failpoint/failpoints.go b/common/failpoint/failpoints.go index c3d31df40..b34d597d2 100644 --- a/common/failpoint/failpoints.go +++ b/common/failpoint/failpoints.go @@ -30,6 +30,7 @@ func newName(s string) Name { var ( PauseBeforeDumping = newName("PauseBeforeDumping") SlowBSONDump = newName("SlowBSONDump") + PauseUntilResumed = newName("PauseUntilResumed") ) // parseNames splits arg, a comma-separated list of failpoint names as diff --git a/mongodump/mongodump.go b/mongodump/mongodump.go index 1acde0a30..695ba1b5c 100644 --- a/mongodump/mongodump.go +++ b/mongodump/mongodump.go @@ -345,6 +345,9 @@ func (dump *MongoDump) Dump() (err error) { if _, ok := failpoint.DefaultManager.Get(failpoint.PauseBeforeDumping); ok { log.Logvf(log.Info, "failpoint.PauseBeforeDumping: sleeping 15 sec") time.Sleep(15 * time.Second) + } else if fp, ok := failpoint.DefaultManager.Get(failpoint.PauseUntilResumed); ok { + log.Logvf(log.Info, "failpoint.PauseUntilResumed: waiting for resume signal") + fp.Wait() } // switch on what kind of execution to do diff --git a/mongodump/oplog_dump_test.go b/mongodump/oplog_dump_test.go index 0cf9398dd..fd9befadf 100644 --- a/mongodump/oplog_dump_test.go +++ b/mongodump/oplog_dump_test.go @@ -546,3 +546,61 @@ func convertIndexToTTL(ctx context.Context, t *testing.T) { ) require.NoError(t, res.Err()) } + +// TestOplogRollover verifies that mongodump refuses to produce a dump when +// the oplog has rolled over during dumping and evicted the entry it captured +// its starting timestamp from. +// +// jstests/dump/oplog_rollover_test.js used to test this by racing a fixed +// 15-second sleep against a fixed amount of write load meant to overflow a +// deliberately tiny (2MB) oplog, and needed several rounds of "make this +// less racy" fixes over the years as host speed varied. That approach isn't +// available here anyway: the shared replica set used by Go integration +// tests can't be resized below 990MB (server-enforced), and forcing genuine +// eviction against that cluster turned out to depend on WiredTiger +// checkpoint/oldest-timestamp scheduling in ways that were empirically +// unpredictable even when forcing checkpoints and using majority write +// concern. +// +// Instead, this uses the PauseUntilResumed failpoint to block mongodump +// after it has captured its real starting timestamp, overwrites that +// timestamp with one guaranteed not to exist in the oplog, and resumes. +// mongodump then runs the exact same overflow-detection logic +// (checkOplogTimestampExists, in oplog_dump.go) it would in production, +// deterministically and without needing to write any data at all. +func TestOplogRollover(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + // Oplog is not available in a standalone topology. + testtype.SkipUnlessTestType(t, testtype.ReplSetTestType) + + md, err := simpleMongoDumpInstance() + require.NoError(t, err) + + md.ToolOptions.DB = "" + md.OutputOptions.Oplog = true + md.OutputOptions.Out = "oplog_rollover" + require.NoError(t, md.Init()) + defer os.RemoveAll(md.OutputOptions.Out) + + require.NoError(t, failpoint.DefaultManager.Parse(failpoint.PauseUntilResumed.String())) + defer failpoint.DefaultManager.Reset() + + dumpErrCh := make(chan error, 1) + go func() { + dumpErrCh <- md.Dump() + }() + + fp, ok := failpoint.DefaultManager.Get(failpoint.PauseUntilResumed) + require.True(t, ok, "PauseUntilResumed failpoint should be enabled") + fp.Reached() + md.oplogStart = bson.Timestamp{T: 1, I: 1} + fp.Signal() + + err = <-dumpErrCh + require.ErrorContains( + t, + err, + "oplog overflow: mongodump was unable to capture all new oplog entries during execution", + "mongodump should crash when the oplog rolls over during dumping", + ) +} diff --git a/test/qa-tests/jstests/dump/oplog_rollover_test.js b/test/qa-tests/jstests/dump/oplog_rollover_test.js deleted file mode 100644 index abb3cfcef..000000000 --- a/test/qa-tests/jstests/dump/oplog_rollover_test.js +++ /dev/null @@ -1,50 +0,0 @@ -(function() { - if (typeof getToolTest === 'undefined') { - load('jstests/configs/replset_single_28_tinyoplog.config.js'); - } - load('jstests/libs/extended_assert.js'); - var assert = extendedAssert; - - var targetPath = 'dump_oplog_rollover_test'; - resetDbpath(targetPath); - var toolTest = getToolTest('oplogRolloverTest'); - var commonToolArgs = getCommonToolArguments(); - - if (!toolTest.isReplicaSet) { - print('Nothing to do for testing oplog rollover without a replica set!'); - return assert(true); - } - - var db = toolTest.db.getSiblingDB('foo'); - - var bigObj = {x: ''}; - while (bigObj.x.length < 1024 * 1024) { - bigObj.x += 'bacon'; - } - - // get collection initialized before we start - db.bar.insert(bigObj); - - var dumpArgs = ['mongodump', - '-vv', - '--oplog', - '--failpoints', 'PauseBeforeDumping', - '--host', toolTest.m.host] - .concat(getDumpTarget(targetPath)) - .concat(commonToolArgs); - - var pid = startMongoProgramNoConnect.apply(null, dumpArgs); - for (var i = 0; i < 1000; ++i) { - db.bar.insert(bigObj); - } - - assert(waitProgram(pid) !== 0, - 'mongodump --oplog should crash sensibly on oplog rollover'); - - var expectedError = 'oplog overflow: mongodump was unable to capture all ' + - 'new oplog entries during execution'; - assert.strContains.soon(expectedError, rawMongoProgramOutput, - 'mongodump --oplog failure should output the correct error message'); - - toolTest.stop(); -}());