Skip to content

Commit c8ad005

Browse files
arvindKandpal-ksolvesarvindKandpal-ksolves
committed
CASSANDRA-21133: Make bin/sstableupgrade functionally on par with nodetool upgradesstables
1 parent dc7935b commit c8ad005

5 files changed

Lines changed: 86 additions & 10 deletions

File tree

‎doc/modules/cassandra/pages/managing/tools/sstable/sstableupgrade.adoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ sstableupgrade <options> <keyspace> <table> [snapshot_name]
2525
|===
2626
|--debug |display stack traces
2727
|-h,--help |display this help message
28+
|-a,--include-all-sstables |include all sstables, even those already on the current version
2829
|-k,--keep-source |do not delete the source sstables
2930
|===
3031

‎src/java/org/apache/cassandra/db/ColumnFamilyStore.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,12 +1025,21 @@ public Descriptor newSSTableDescriptor(File directory, SSTableFormat<?, ?> forma
10251025

10261026
public Descriptor newSSTableDescriptor(File directory, Version version)
10271027
{
1028-
Descriptor newDescriptor = new Descriptor(version,
1029-
directory,
1030-
getKeyspaceName(),
1031-
name,
1032-
sstableIdGenerator.get());
1033-
assert !newDescriptor.fileFor(Components.DATA).exists();
1028+
Descriptor newDescriptor;
1029+
while (true)
1030+
{
1031+
newDescriptor = new Descriptor(version,
1032+
directory,
1033+
getKeyspaceName(),
1034+
name,
1035+
sstableIdGenerator.get());
1036+
1037+
if (!newDescriptor.fileFor(Components.DATA).exists())
1038+
break;
1039+
1040+
logger.warn("Generated SSTable id {} collides with existing file; advancing.", newDescriptor.id);
1041+
}
1042+
10341043
return newDescriptor;
10351044
}
10361045

‎src/java/org/apache/cassandra/tools/StandaloneUpgrader.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ public class StandaloneUpgrader
5353
private static final String DEBUG_OPTION = "debug";
5454
private static final String HELP_OPTION = "help";
5555
private static final String KEEP_SOURCE = "keep-source";
56+
private static final String INCLUDE_ALL = "include-all-sstables";
5657

5758
public static void main(String args[])
5859
{
@@ -91,7 +92,7 @@ public static void main(String args[])
9192
try
9293
{
9394
SSTableReader sstable = SSTableReader.openNoValidation(entry.getKey(), components, cfs);
94-
if (sstable.descriptor.version.equals(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()))
95+
if (!options.includeAll && sstable.descriptor.version.equals(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()))
9596
{
9697
sstable.selfRef().release();
9798
continue;
@@ -151,6 +152,7 @@ private static class Options
151152

152153
public boolean debug;
153154
public boolean keepSource;
155+
public boolean includeAll;
154156

155157
private Options(String keyspace, String cf, String snapshot)
156158
{
@@ -191,6 +193,7 @@ public static Options parseArgs(String cmdArgs[])
191193

192194
opts.debug = cmd.hasOption(DEBUG_OPTION);
193195
opts.keepSource = cmd.hasOption(KEEP_SOURCE);
196+
opts.includeAll = cmd.hasOption(INCLUDE_ALL);
194197

195198
return opts;
196199
}
@@ -214,6 +217,7 @@ private static CmdLineOptions getCmdLineOptions()
214217
options.addOption(null, DEBUG_OPTION, "display stack traces");
215218
options.addOption("h", HELP_OPTION, "display this help message");
216219
options.addOption("k", KEEP_SOURCE, "do not delete the source sstables");
220+
options.addOption("a", "include-all-sstables", "include all sstables, even those already on the current version");
217221
return options;
218222
}
219223

‎test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.cassandra.SchemaLoader;
4949
import org.apache.cassandra.UpdateBuilder;
5050
import org.apache.cassandra.Util;
51+
import org.apache.cassandra.config.DatabaseDescriptor;
5152
import org.apache.cassandra.cql3.ColumnIdentifier;
5253
import org.apache.cassandra.cql3.Operator;
5354
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
@@ -71,8 +72,11 @@
7172
import org.apache.cassandra.index.transactions.UpdateTransaction;
7273
import org.apache.cassandra.io.sstable.Component;
7374
import org.apache.cassandra.io.sstable.Descriptor;
75+
import org.apache.cassandra.io.sstable.SSTableIdFactory;
76+
import org.apache.cassandra.io.sstable.SSTableId;
7477
import org.apache.cassandra.io.sstable.SSTableReadsListener;
7578
import org.apache.cassandra.io.sstable.ScrubTest;
79+
import org.apache.cassandra.io.sstable.format.Version;
7680
import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
7781
import org.apache.cassandra.io.sstable.format.SSTableReader;
7882
import org.apache.cassandra.io.util.File;
@@ -839,6 +843,52 @@ public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot)
839843
return schemaAndManifestFileSizes;
840844
}
841845

846+
@Test
847+
public void testNewSSTableDescriptorCollision() throws Exception
848+
{
849+
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
850+
File dir = cfs.getDirectories().getDirectoryForNewSSTables();
851+
Version version = DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion();
852+
853+
Descriptor probe = cfs.newSSTableDescriptor(dir, version);
854+
855+
int currentId;
856+
try {
857+
currentId = Integer.parseInt(probe.id.toString());
858+
} catch (NumberFormatException e) {
859+
return;
860+
}
861+
862+
SSTableId blockedId1 = SSTableIdFactory.instance.fromString(String.valueOf(currentId + 1));
863+
SSTableId blockedId2 = SSTableIdFactory.instance.fromString(String.valueOf(currentId + 2));
864+
865+
Descriptor blockedDesc1 = new Descriptor(version, dir, cfs.getKeyspaceName(), cfs.name, blockedId1);
866+
Descriptor blockedDesc2 = new Descriptor(version, dir, cfs.getKeyspaceName(), cfs.name, blockedId2);
867+
868+
File blockedDataFile1 = blockedDesc1.fileFor(Components.DATA);
869+
File blockedDataFile2 = blockedDesc2.fileFor(Components.DATA);
870+
871+
try
872+
{
873+
blockedDataFile1.createFileIfNotExists();
874+
blockedDataFile2.createFileIfNotExists();
875+
876+
Descriptor nextDesc = cfs.newSSTableDescriptor(dir, version);
877+
878+
Assert.assertNotEquals("The loop should have skipped the first blocked ID",
879+
blockedDesc1.id, nextDesc.id);
880+
Assert.assertNotEquals("The loop should have skipped the second blocked ID",
881+
blockedDesc2.id, nextDesc.id);
882+
Assert.assertFalse("The returned descriptor must not already exist on disk",
883+
nextDesc.fileFor(Components.DATA).exists());
884+
}
885+
finally
886+
{
887+
blockedDataFile1.deleteIfExists();
888+
blockedDataFile2.deleteIfExists();
889+
}
890+
}
891+
842892
private Memtable fakeMemTableWithMinTS(ColumnFamilyStore cfs, long minTS)
843893
{
844894
return new AbstractMemtable(cfs.metadata, minTS)

‎test/unit/org/apache/cassandra/tools/StandaloneUpgraderTest.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,11 @@ public void testMaybeChangeDocs()
4848
"hard links to live sstables.\n" +
4949
"--\n" +
5050
"Options are:\n" +
51-
" --debug display stack traces\n" +
52-
" -h,--help display this help message\n" +
53-
" -k,--keep-source do not delete the source sstables\n";
51+
" -a,--include-all-sstables include all sstables, even those already on the\n" +
52+
" current version\n" +
53+
" --debug display stack traces\n" +
54+
" -h,--help display this help message\n" +
55+
" -k,--keep-source do not delete the source sstables\n";
5456
Assertions.assertThat(tool.getStdout()).isEqualTo(help);
5557
}
5658

@@ -86,6 +88,16 @@ public void testFlagArgs()
8688
assertEquals(0, tool.getExitCode());
8789
assertCorrectEnvPostTest();
8890
});
91+
92+
Arrays.asList("-a", "--include-all-sstables").forEach(arg -> {
93+
ToolResult tool = ToolRunner.invokeClass(StandaloneUpgrader.class,
94+
arg,
95+
"system_schema",
96+
"tables");
97+
Assertions.assertThat(tool.getCleanedStderr()).as("Arg: [%s]", arg).isEmpty();
98+
assertEquals(0, tool.getExitCode());
99+
assertCorrectEnvPostTest();
100+
});
89101
}
90102

91103
@Test

0 commit comments

Comments
 (0)