Skip to content

Commit 2191111

Browse files
zombachuzombachu
committed
Improve migration command
1. Migrates data in worlds/ and groups/ even if the player file doesn't have a matching player in players/ (possible due to legacy versions mixing and matching UUID and username-based file names) 2. Processes players sequentially to avoid resource exhaustion with thousands of files 3. Adds per-profile and per-player exception handling to avoid errors cancelling the whole migration 4. Continues on serialization errors to avoid invalid items skipping the whole inventory
1 parent 7642411 commit 2191111

3 files changed

Lines changed: 128 additions & 34 deletions

File tree

‎src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java‎

Lines changed: 123 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.mvplugins.multiverse.inventories.commands.bulkedit.playerprofile;
22

3+
import com.google.common.io.Files;
34
import org.jvnet.hk2.annotations.Service;
45
import org.mvplugins.multiverse.core.command.MVCommandIssuer;
56
import org.mvplugins.multiverse.core.command.queue.CommandQueueManager;
@@ -12,13 +13,25 @@
1213
import org.mvplugins.multiverse.inventories.commands.InventoriesCommand;
1314
import org.mvplugins.multiverse.inventories.config.InventoriesConfig;
1415
import org.mvplugins.multiverse.inventories.profile.GlobalProfile;
16+
import org.mvplugins.multiverse.inventories.profile.ProfileCacheManager;
1517
import org.mvplugins.multiverse.inventories.profile.ProfileDataSource;
1618
import org.mvplugins.multiverse.inventories.profile.key.GlobalProfileKey;
1719
import org.mvplugins.multiverse.inventories.profile.key.ProfileKey;
20+
import org.mvplugins.multiverse.inventories.profile.key.ProfileType;
1821
import org.mvplugins.multiverse.inventories.profile.key.ProfileTypes;
1922
import org.mvplugins.multiverse.inventories.profile.key.ContainerType;
23+
import org.mvplugins.multiverse.inventories.profile.ProfileFilesLocator;
24+
import org.mvplugins.multiverse.inventories.util.ItemStackConverter;
2025

21-
import java.util.Arrays;
26+
import java.io.File;
27+
import java.nio.charset.StandardCharsets;
28+
import java.util.ArrayList;
29+
import java.util.HashMap;
30+
import java.util.HashSet;
31+
import java.util.List;
32+
import java.util.Map;
33+
import java.util.Set;
34+
import java.util.UUID;
2235
import java.util.concurrent.CompletableFuture;
2336
import java.util.concurrent.atomic.AtomicLong;
2437

@@ -28,21 +41,32 @@ final class MigrateInventorySerializationCommand extends InventoriesCommand {
2841
private final CommandQueueManager commandQueueManager;
2942
private final ProfileDataSource profileDataSource;
3043
private final InventoriesConfig inventoriesConfig;
44+
private final ProfileCacheManager profileCacheManager;
45+
private final ProfileFilesLocator profileFilesLocator;
3146

3247
@Inject
3348
MigrateInventorySerializationCommand(
3449
@NotNull CommandQueueManager commandQueueManager,
3550
@NotNull ProfileDataSource profileDataSource,
36-
@NotNull InventoriesConfig inventoriesConfig
51+
@NotNull InventoriesConfig inventoriesConfig,
52+
@NotNull ProfileCacheManager profileCacheManager,
53+
@NotNull ProfileFilesLocator profileFilesLocator
3754
) {
3855
this.commandQueueManager = commandQueueManager;
3956
this.profileDataSource = profileDataSource;
4057
this.inventoriesConfig = inventoriesConfig;
58+
this.profileCacheManager = profileCacheManager;
59+
this.profileFilesLocator = profileFilesLocator;
4160
}
4261

4362
@Subcommand("bulkedit migrate inventory-serialization nbt")
4463
@CommandPermission("multiverse.inventories.bulkedit")
4564
void onNbtCommand(MVCommandIssuer issuer) {
65+
if (!ItemStackConverter.hasByteSerializeSupport()) {
66+
issuer.sendMessage("NBT serialization is only supported on PaperMC 1.20.2 or higher!");
67+
issuer.sendMessage("Conversion to NBT is not possible on your current server version.");
68+
return;
69+
}
4670
commandQueueManager.addToQueue(CommandQueuePayload.issuer(issuer)
4771
.prompt(Message.of("Are you sure you want to migrate all player data to NBT?"))
4872
.action(() -> doMigration(issuer, true)));
@@ -57,44 +81,114 @@ void onBukkitCommand(MVCommandIssuer issuer) {
5781
}
5882

5983
private void doMigration(MVCommandIssuer issuer, boolean useByteSerialization) {
84+
issuer.sendMessage("Updating config and clearing caches...");
6085
inventoriesConfig.setUseByteSerializationForInventoryData(useByteSerialization);
6186
inventoriesConfig.save();
87+
profileCacheManager.clearAllCache();
6288

6389
long startTime = System.nanoTime();
6490
AtomicLong profileCounter = new AtomicLong(0);
65-
CompletableFuture.allOf(profileDataSource.listGlobalProfileUUIDs()
66-
.stream()
67-
.map(playerUUID -> profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, ""))
68-
.thenCompose(profile -> run(profile, profileCounter))
69-
.exceptionally(throwable -> {
70-
issuer.sendMessage("Error updating player " + playerUUID + ": " + throwable.getMessage());
71-
return null;
72-
}))
73-
.toArray(CompletableFuture[]::new))
91+
92+
Map<ContainerType, List<String>> containerNames = new HashMap<>();
93+
for (ContainerType type : ContainerType.values()) {
94+
containerNames.put(type, profileDataSource.listContainerDataNames(type));
95+
}
96+
97+
Set<String> playerIdentifiers = new HashSet<>();
98+
// Scan global files
99+
profileFilesLocator.listGlobalFiles().forEach(file ->
100+
playerIdentifiers.add(Files.getNameWithoutExtension(file.getName())));
101+
102+
// Scan world and group files
103+
for (ContainerType type : ContainerType.values()) {
104+
for (File folder : profileFilesLocator.listProfileContainerFolders(type)) {
105+
profileFilesLocator.listPlayerProfileFiles(type, folder.getName()).forEach(file ->
106+
playerIdentifiers.add(Files.getNameWithoutExtension(file.getName())));
107+
}
108+
}
109+
110+
issuer.sendMessage("Found " + playerIdentifiers.size() + " unique player identifiers to migrate.");
111+
112+
migrateNextPlayer(issuer, new ArrayList<>(playerIdentifiers), 0, containerNames, profileCounter)
74113
.thenRun(() -> {
75114
long timeDuration = (System.nanoTime() - startTime) / 1000000;
76115
issuer.sendMessage("Updated " + profileCounter.get() + " player profiles.");
77116
issuer.sendMessage("Bulk edit completed in " + timeDuration + " ms.");
78117
});
79118
}
80119

81-
private CompletableFuture<Void> run(GlobalProfile profile, AtomicLong profileCounter) {
82-
return CompletableFuture.allOf(Arrays.stream(ContainerType.values())
83-
.flatMap(containerType -> profileDataSource.listContainerDataNames(containerType).stream()
84-
.flatMap(dataName -> ProfileTypes.getTypes().stream()
85-
.map(profileType -> profileDataSource.getPlayerProfile(ProfileKey.of(
86-
containerType,
87-
dataName,
88-
profileType,
89-
profile.getPlayerUUID(),
90-
profile.getLastKnownName()
91-
)).thenCompose(playerProfile -> {
92-
if (playerProfile.getData().isEmpty()) {
93-
return CompletableFuture.completedFuture(null);
94-
}
95-
profileCounter.incrementAndGet();
96-
return profileDataSource.updatePlayerProfile(playerProfile);
97-
}))))
98-
.toArray(CompletableFuture[]::new));
120+
private CompletableFuture<Void> migrateNextPlayer(
121+
MVCommandIssuer issuer,
122+
List<String> playerIdentifiers,
123+
int index,
124+
Map<ContainerType, List<String>> containerNames,
125+
AtomicLong profileCounter
126+
) {
127+
if (index >= playerIdentifiers.size()) {
128+
return CompletableFuture.completedFuture(null);
129+
}
130+
131+
String playerIdentifier = playerIdentifiers.get(index);
132+
UUID playerUUID;
133+
try {
134+
playerUUID = UUID.fromString(playerIdentifier);
135+
} catch (IllegalArgumentException e) {
136+
playerUUID = UUID.nameUUIDFromBytes(("OfflinePlayer:" + playerIdentifier).getBytes(StandardCharsets.UTF_8));
137+
}
138+
139+
if (index % 50 == 0 && index > 0) {
140+
issuer.sendMessage("Processed " + index + " players...");
141+
}
142+
143+
return profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, playerIdentifier))
144+
.thenCompose(profile -> run(issuer, profile, containerNames, profileCounter))
145+
.exceptionally(throwable -> {
146+
issuer.sendMessage("Error updating player " + playerIdentifier + ": " + throwable.getMessage());
147+
return null;
148+
})
149+
.thenCompose(v -> migrateNextPlayer(issuer, playerIdentifiers, index + 1, containerNames, profileCounter));
150+
}
151+
152+
private CompletableFuture<Void> run(
153+
MVCommandIssuer issuer,
154+
GlobalProfile profile,
155+
Map<ContainerType, List<String>> containerNames,
156+
AtomicLong profileCounter
157+
) {
158+
String playerName = profile.getLastKnownName();
159+
if (playerName == null || playerName.isEmpty()) {
160+
playerName = profile.getPlayerUUID().toString();
161+
}
162+
163+
CompletableFuture<Void> future = CompletableFuture.completedFuture(null);
164+
for (ContainerType containerType : ContainerType.values()) {
165+
List<String> dataNames = containerNames.get(containerType);
166+
for (String dataName : dataNames) {
167+
for (ProfileType profileType : ProfileTypes.getTypes()) {
168+
ProfileKey profileKey = ProfileKey.of(
169+
containerType,
170+
dataName,
171+
profileType,
172+
profile.getPlayerUUID(),
173+
playerName
174+
);
175+
future = future.thenCompose(v -> profileDataSource.getPlayerProfile(profileKey)
176+
.thenCompose(playerProfile -> {
177+
if (playerProfile.getData().isEmpty()) {
178+
return CompletableFuture.completedFuture(null);
179+
}
180+
profileCounter.incrementAndGet();
181+
return profileDataSource.updatePlayerProfile(playerProfile);
182+
})
183+
.exceptionally(throwable -> {
184+
issuer.sendMessage(String.format("Error migrating profile %s %s/%s for player %s: %s",
185+
containerType, dataName, profileType, profile.getPlayerUUID(), throwable.getMessage()));
186+
return null;
187+
})
188+
);
189+
}
190+
}
191+
}
192+
return future;
99193
}
100194
}

‎src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ final class FlatFileProfileDataSource implements ProfileDataSource {
5757

5858
private FileConfiguration loadFileToJsonConfiguration(File file) {
5959
JsonConfiguration jsonConfiguration = new JsonConfiguration();
60-
jsonConfiguration.options().continueOnSerializationError(false);
60+
jsonConfiguration.options().continueOnSerializationError(true);
6161
Try.run(() -> jsonConfiguration.load(file)).getOrElseThrow(e -> {
6262
Logging.severe("Could not load file %s : %s", file, e.getMessage());
6363
e.printStackTrace();

‎src/main/java/org/mvplugins/multiverse/inventories/profile/ProfileFilesLocator.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import java.util.UUID;
1919

2020
@Service
21-
final class ProfileFilesLocator {
21+
public final class ProfileFilesLocator {
2222

2323
private static final String JSON = ".json";
2424

@@ -67,7 +67,7 @@ File getContainerFolder(ContainerType type) {
6767
};
6868
}
6969

70-
List<File> listProfileContainerFolders(ContainerType type) {
70+
public List<File> listProfileContainerFolders(ContainerType type) {
7171
return Option.of(getContainerFolder(type).listFiles())
7272
.map(filesList -> Arrays.stream(filesList)
7373
.filter(File::isDirectory)
@@ -83,7 +83,7 @@ File getProfileContainerFolder(ContainerType type, String folderName) {
8383
return folder;
8484
}
8585

86-
List<File> listPlayerProfileFiles(ContainerType type, String dataName) {
86+
public List<File> listPlayerProfileFiles(ContainerType type, String dataName) {
8787
return Option.of(getProfileContainerFolder(type, dataName).listFiles())
8888
.map(filesList -> Arrays.stream(filesList)
8989
.filter(File::isFile)
@@ -120,7 +120,7 @@ File getGlobalFolder() {
120120
return this.globalFolder;
121121
}
122122

123-
List<File> listGlobalFiles() {
123+
public List<File> listGlobalFiles() {
124124
return Option.of(this.globalFolder.listFiles())
125125
.map(filesList -> Arrays.stream(filesList)
126126
.filter(File::isFile)

0 commit comments

Comments
 (0)