Skip to content

Commit d5025fd

Browse files
roachtest: add split-file import test
Add an import roachtest that splits a dataset's data files into two halves and imports them via two separate IMPORT jobs into the same table. This verifies that successive imports produce a correct, complete dataset. Epic: None Release note: None Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>
1 parent 9fb030e commit d5025fd

1 file changed

Lines changed: 43 additions & 0 deletions

File tree

pkg/cmd/roachtest/tests/import.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,13 @@ var tests = []importTestSpec{
337337
datasetNames: FromFunc(anyDataset),
338338
importRunner: importPauseRunner,
339339
},
340+
// Test importing a table in two separate IMPORT jobs (split files).
341+
{
342+
subtestName: "split",
343+
nodes: []int{4},
344+
datasetNames: FromFunc(anyDataset),
345+
importRunner: splitImportRunner,
346+
},
340347
}
341348

342349
// importTestTimeout is the timeout for import roachtests. This is
@@ -860,6 +867,42 @@ func importPauseRunner(
860867
return nil
861868
}
862869

870+
// splitImportRunner imports a table's data in two separate IMPORT jobs,
871+
// splitting the data files into two halves. This verifies that successive
872+
// imports into the same table produce a correct, complete dataset.
873+
func splitImportRunner(
874+
ctx context.Context, t test.Test, c cluster.Cluster, l *logger.Logger, _ *rand.Rand, ds dataset,
875+
) error {
876+
conn := c.Conn(ctx, l, 1)
877+
defer conn.Close()
878+
879+
urls := ds.getDataURLs()
880+
var first, second []string
881+
for i, url := range urls {
882+
if i%2 == 0 {
883+
first = append(first, url)
884+
} else {
885+
second = append(second, url)
886+
}
887+
}
888+
889+
t.WorkerStatus(fmt.Sprintf("importing first half (%d files) of %s",
890+
len(first), ds.getTableName()))
891+
importStmt := formatImportStmt(ds.getTableName(), first, false)
892+
if _, err := conn.ExecContext(ctx, importStmt); err != nil {
893+
return errors.Wrap(err, "first import")
894+
}
895+
896+
t.WorkerStatus(fmt.Sprintf("importing second half (%d files) of %s",
897+
len(second), ds.getTableName()))
898+
importStmt = formatImportStmt(ds.getTableName(), second, false)
899+
if _, err := conn.ExecContext(ctx, importStmt); err != nil {
900+
return errors.Wrap(err, "second import")
901+
}
902+
903+
return nil
904+
}
905+
863906
// makeColumnFamilies() is a pre-test hook that changes the tables
864907
// in import_test to use column families. To do this, we iterate the
865908
// tables in the database, reading the schema for each table, modifying

0 commit comments

Comments
 (0)