diff --git a/build.gradle b/build.gradle index 2bf0a0966b5..0cd233502cd 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,10 @@ dependencies { exclude group: 'org.bukkit', module: 'bukkit' } + // Libraries that are installed at runtime. See plugin.yml 'libraries' section. + implementation group: 'com.h2database', name: 'h2', version: project.property('h2.version') + implementation group: 'com.zaxxer', name: 'HikariCP', version: project.property('hikaricp.version') + implementation fileTree(dir: 'lib', include: '*.jar') testShadow group: 'junit', name: 'junit', version: '4.13.2' @@ -311,6 +315,15 @@ if (project.hasProperty('junit') && (project.property('junit') as String).toLowe createTestTask('customTest', 'Runs tests based on provided parameters.', customEnvironments, envJava, customTimeout, customModifiers.toArray(new Modifiers[0]) as Modifiers[]) // end custom test task +// Generic replace tokens, e.g: '@version@' +tasks.withType(Copy).configureEach { + filter(ReplaceTokens, tokens: [ + 'today' : '' + LocalTime.now(), + 'h2.version' : project.property('h2.version'), + 'hikaricp.version' : project.property('hikaricp.version') + ]) +} + // Build flavor configurations tasks.register('githubResources', ProcessResources) { from 'src/main/resources', { @@ -321,7 +334,6 @@ tasks.register('githubResources', ProcessResources) { channel = 'prerelease' filter ReplaceTokens, tokens: [ 'version' : version, - 'today' : '' + LocalTime.now(), 'release-flavor' : 'skriptlang-github', // SkriptLang build, distributed on Github 'release-channel' : channel, // Release channel, see above 'release-updater' : 'ch.njol.skript.update.GithubChecker', // Github API client @@ -354,7 +366,6 @@ tasks.register('spigotResources', ProcessResources) { channel = 'prerelease' filter ReplaceTokens, tokens: [ 'version' : version, - 'today' : '' + LocalTime.now(), 'release-flavor' : 'skriptlang-spigot', // SkriptLang build, distributed on Spigot resources 'release-channel' : channel, // Release channel, see above 'release-updater' : 'ch.njol.skript.update.GithubChecker', // Github API client @@ -385,7 +396,6 @@ tasks.register('nightlyResources', ProcessResources) { version = project.property('version') + '-nightly-' + hash filter ReplaceTokens, tokens: [ 'version' : version, - 'today' : '' + LocalTime.now(), 'release-flavor' : 'skriptlang-nightly', // SkriptLang build, automatically done by CI 'release-channel' : 'prerelease', // No update checking, but these are VERY unstable 'release-updater' : 'ch.njol.skript.update.NoUpdateChecker', // No auto updates for now diff --git a/gradle.properties b/gradle.properties index 6f4caa5668f..af67f4ca6ac 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,3 +9,6 @@ version=2.14.0 jarName=Skript.jar testEnv=java21/paper-1.21.11 testEnvJavaVersion=21 + +hikaricp.version=5.1.0 +h2.version=2.3.232 diff --git a/lib/SQLibrary-7.1.jar b/lib/SQLibrary-7.1.jar deleted file mode 100644 index e0507266ce8..00000000000 Binary files a/lib/SQLibrary-7.1.jar and /dev/null differ diff --git a/src/main/java/ch/njol/skript/SkriptAddon.java b/src/main/java/ch/njol/skript/SkriptAddon.java index 82137f507a5..17530c0df43 100644 --- a/src/main/java/ch/njol/skript/SkriptAddon.java +++ b/src/main/java/ch/njol/skript/SkriptAddon.java @@ -11,6 +11,10 @@ import ch.njol.skript.util.Utils; import ch.njol.skript.util.Version; +import ch.njol.skript.variables.VariableStorage; +import ch.njol.skript.variables.Variables; + +import org.jetbrains.annotations.ApiStatus; import org.skriptlang.skript.localization.Localizer; import org.skriptlang.skript.registration.SyntaxRegistry; import org.skriptlang.skript.util.Registry; diff --git a/src/main/java/ch/njol/skript/effects/Delay.java b/src/main/java/ch/njol/skript/effects/Delay.java index 16f4c50780f..5946c51bb6f 100644 --- a/src/main/java/ch/njol/skript/effects/Delay.java +++ b/src/main/java/ch/njol/skript/effects/Delay.java @@ -14,6 +14,7 @@ import ch.njol.skript.timings.SkriptTimings; import ch.njol.skript.util.Timespan; import ch.njol.skript.variables.Variables; +import ch.njol.skript.variables.VariablesMap; import ch.njol.util.Kleenean; import org.bukkit.Bukkit; import org.bukkit.event.Event; @@ -72,7 +73,7 @@ protected TriggerItem walk(Event event) { return null; // Back up local variables - Object localVars = Variables.removeLocals(event); + VariablesMap localVars = Variables.removeLocals(event); Bukkit.getScheduler().scheduleSyncDelayedTask(Skript.getInstance(), () -> { addDelayedEvent(event); diff --git a/src/main/java/ch/njol/skript/effects/EffTeleport.java b/src/main/java/ch/njol/skript/effects/EffTeleport.java index 145cc5f57f5..db24a3cd41f 100644 --- a/src/main/java/ch/njol/skript/effects/EffTeleport.java +++ b/src/main/java/ch/njol/skript/effects/EffTeleport.java @@ -9,6 +9,7 @@ import ch.njol.skript.timings.SkriptTimings; import ch.njol.skript.util.Direction; import ch.njol.skript.variables.Variables; +import ch.njol.skript.variables.VariablesMap; import ch.njol.util.Kleenean; import io.papermc.lib.PaperLib; import io.papermc.lib.environments.PaperEnvironment; @@ -131,7 +132,7 @@ public boolean init(Expression[] exprs, int matchedPattern, Kleenean isDelaye } final Location fixed = location; - Object localVars = Variables.removeLocals(event); + VariablesMap localVars = Variables.removeLocals(event); // This will either fetch the chunk instantly if on Spigot or already loaded or fetch it async if on Paper. PaperLib.getChunkAtAsync(location).thenAccept(chunk -> { diff --git a/src/main/java/ch/njol/skript/effects/IndeterminateDelay.java b/src/main/java/ch/njol/skript/effects/IndeterminateDelay.java index c3d11530e0d..7c63bc40809 100644 --- a/src/main/java/ch/njol/skript/effects/IndeterminateDelay.java +++ b/src/main/java/ch/njol/skript/effects/IndeterminateDelay.java @@ -1,5 +1,6 @@ package ch.njol.skript.effects; +import ch.njol.skript.variables.VariablesMap; import org.bukkit.Bukkit; import org.bukkit.event.Event; import org.jetbrains.annotations.Nullable; @@ -28,7 +29,7 @@ protected TriggerItem walk(Event event) { return null; // Back up local variables - Object localVars = Variables.removeLocals(event); + VariablesMap localVars = Variables.removeLocals(event); Bukkit.getScheduler().scheduleSyncDelayedTask(Skript.getInstance(), () -> { Delay.addDelayedEvent(event); diff --git a/src/main/java/ch/njol/skript/lang/Variable.java b/src/main/java/ch/njol/skript/lang/Variable.java index 1c7b127874f..9339798b143 100644 --- a/src/main/java/ch/njol/skript/lang/Variable.java +++ b/src/main/java/ch/njol/skript/lang/Variable.java @@ -1,5 +1,12 @@ package ch.njol.skript.lang; +import java.lang.reflect.Array; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Pattern; +import java.util.function.Predicate; +import java.util.function.Function; + import ch.njol.skript.Skript; import ch.njol.skript.SkriptAPIException; import ch.njol.skript.SkriptConfig; @@ -7,6 +14,10 @@ import ch.njol.skript.classes.Changer.ChangeMode; import ch.njol.skript.classes.Changer.ChangerUtils; import ch.njol.skript.classes.ClassInfo; +import ch.njol.skript.variables.VariableStorage; +import org.skriptlang.skript.lang.arithmetic.Arithmetics; +import org.skriptlang.skript.lang.arithmetic.OperationInfo; +import org.skriptlang.skript.lang.arithmetic.Operator; import ch.njol.skript.lang.SkriptParser.ParseResult; import ch.njol.skript.lang.parser.ParserInstance; import ch.njol.skript.lang.util.SimpleExpression; @@ -27,21 +38,12 @@ import org.bukkit.event.Event; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.skriptlang.skript.lang.arithmetic.Arithmetics; -import org.skriptlang.skript.lang.arithmetic.OperationInfo; -import org.skriptlang.skript.lang.arithmetic.Operator; import org.skriptlang.skript.lang.comparator.Comparators; import org.skriptlang.skript.lang.comparator.Relation; import org.skriptlang.skript.lang.converter.Converters; import org.skriptlang.skript.lang.script.Script; import org.skriptlang.skript.lang.script.ScriptWarning; -import java.lang.reflect.Array; -import java.util.*; -import java.util.Map.Entry; -import java.util.function.Function; -import java.util.function.Predicate; - public class Variable implements Expression, KeyReceiverExpression, KeyProviderExpression { private final static String SINGLE_SEPARATOR_CHAR = ":"; @@ -403,7 +405,7 @@ public Iterator> keyedIterator(Event event) { return Iterators.filter(transformed, Objects::nonNull); } - public Iterator> variablesIterator(Event event) { + public Iterator> variablesIterator(Event event) { if (!list) throw new SkriptAPIException("Looping a non-list variable"); return Variables.getVariableIterator(name.toString(event), local, event); diff --git a/src/main/java/ch/njol/skript/registrations/Classes.java b/src/main/java/ch/njol/skript/registrations/Classes.java index ad3fba89b00..5856b48224a 100644 --- a/src/main/java/ch/njol/skript/registrations/Classes.java +++ b/src/main/java/ch/njol/skript/registrations/Classes.java @@ -16,20 +16,18 @@ import ch.njol.skript.log.SkriptLogger; import ch.njol.skript.util.StringMode; import ch.njol.skript.util.Utils; -import ch.njol.skript.variables.SQLStorage; +import ch.njol.skript.variables.JdbcStorage; import ch.njol.skript.variables.SerializedVariable; import ch.njol.skript.variables.Variables; import ch.njol.util.Kleenean; import ch.njol.util.StringUtils; import ch.njol.yggdrasil.Tag; import ch.njol.yggdrasil.Yggdrasil; -import ch.njol.yggdrasil.YggdrasilInputStream; import ch.njol.yggdrasil.YggdrasilOutputStream; import com.google.common.base.Preconditions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.bukkit.Bukkit; import org.bukkit.ChatColor; -import org.bukkit.Chunk; import org.jetbrains.annotations.*; import org.skriptlang.skript.lang.converter.Converter; import org.skriptlang.skript.lang.converter.ConverterInfo; @@ -40,6 +38,8 @@ import java.lang.reflect.Array; import java.nio.charset.Charset; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; /** @@ -67,8 +67,8 @@ public static void registerClass(final ClassInfo info) { throw new IllegalArgumentException("Can't register " + info.getC().getName() + " with the code name " + info.getCodeName() + " because that name is already used by " + classInfosByCodeName.get(info.getCodeName())); if (exactClassInfos.containsKey(info.getC())) throw new IllegalArgumentException("Can't register the class info " + info.getCodeName() + " because the class " + info.getC().getName() + " is already registered"); - if (info.getCodeName().length() > SQLStorage.MAX_CLASS_CODENAME_LENGTH) - throw new IllegalArgumentException("The codename '" + info.getCodeName() + "' is too long to be saved in a database, the maximum length allowed is " + SQLStorage.MAX_CLASS_CODENAME_LENGTH); + if (info.getCodeName().length() > JdbcStorage.MAX_CLASS_CODENAME_LENGTH) + throw new IllegalArgumentException("The codename '" + info.getCodeName() + "' is too long to be saved in a database, the maximum length allowed is " + JdbcStorage.MAX_CLASS_CODENAME_LENGTH); exactClassInfos.put(info.getC(), info); classInfosByCodeName.put(info.getCodeName(), info); tempClassInfos.add(info); @@ -743,16 +743,32 @@ private static byte[] getYggdrasilStart(final ClassInfo c) throws NotSerializ } /** - * Must be called on the appropriate thread for the given value (i.e. the main thread currently) + * Represents a context for serialization of a value as a variable. + * + * @param classInfo class info of the object + * @param value object to serialize */ - public static SerializedVariable.@Nullable Value serialize(@Nullable Object object) { - if (object == null) - return null; + private record SerializationContext(ClassInfo classInfo, Object value) { + public @Nullable Serializer serializer() { + return classInfo.getSerializer(); + } + public boolean mustSyncDeserialization() { + Serializer serializer = serializer(); + return serializer != null && serializer.mustSyncDeserialization(); + } + } - // temporary - assert Bukkit.isPrimaryThread(); - + /** + * Returns the serializer used for serializing the given object as a variable. + *

+ * Returns {@code null} if the object can not be serialized (there is no serializer available). + * + * @param object object to serialize + * @return serializer for the serialization of given object + */ + private static SerializationContext getSerializationContext(Object object) { ClassInfo classInfo = getSuperClassInfo(object.getClass()); + if (classInfo.getSerializeAs() != null) { classInfo = getExactClassInfo(classInfo.getSerializeAs()); if (classInfo == null) { @@ -765,101 +781,313 @@ private static byte[] getYggdrasilStart(final ClassInfo c) throws NotSerializ return null; } } - - Serializer serializer = classInfo.getSerializer(); - if (serializer == null) // value cannot be saved + return new SerializationContext(classInfo, object); + } + + /** + * Serializes the provided map of variables. + *

+ * Is blocking if the serializer for some of the variables needs to be synchronized and + * the method is not called from the main thread. + *

+ * This does processed null values in the map and will provide empty + * serialized variables in the returned set for such variables. + *

+ * This method is thread safe. + * + * @param variables variables to serialize + * @return serialized variables, returns null if the serialization failed + * because Skript is disabled, some of the variables need to be serialized + * on the main thread and this method was called off the main thread. + */ + @Blocking + public static @Nullable Set serialize(Map variables) { + Set collected = ConcurrentHashMap.newKeySet(); + Map needsSync = new ConcurrentHashMap<>(); + + variables.entrySet().parallelStream().forEach(entry -> { + String key = entry.getKey(); + Object value = entry.getValue(); + + if (value == null) { + collected.add(new SerializedVariable(key, null)); + return; + } + + SerializationContext context = getSerializationContext(value); + assert context != null; + if (context.classInfo.getSerializer() == null) { + collected.add(new SerializedVariable(key, null)); + return; + } + + if (context.mustSyncDeserialization()) { + needsSync.put(key, context); + return; + } + try { + var serialized = serialize(context.value, context.classInfo); + collected.add(new SerializedVariable(key, serialized)); + } catch (IOException exception) { + Skript.error("Failed to serialize " + context.value); + } + }); + + Runnable syncSerialization = () -> needsSync.forEach((key, context) -> { + try { + var serialized = serialize(context.value, context.classInfo); + collected.add(new SerializedVariable(key, serialized)); + } catch (IOException exception) { + Skript.exception(exception, "Failed to serialize " + context.value); + } + }); + + if (needsSync.isEmpty()) + return collected; + + if (Bukkit.isPrimaryThread()) { + syncSerialization.run(); + } else { + try { + if (!Skript.getInstance().isEnabled()) + // At this point we can not serialize variables synchronously, + // we fail rather than provide partial result + return null; + CompletableFuture.supplyAsync(() -> { + syncSerialization.run(); + return null; + }, Bukkit.getScheduler().getMainThreadExecutor(Skript.getInstance())).get(); + } catch (Exception exception) { + Skript.exception(exception, "Failed to serialize variables on the main thread"); + } + } + + return collected; + } + + /** + * Serializes the provided object to a value for a variable. + *

+ * Is blocking if the serializer for the action needs to be synchronized and + * the method is not called from the main thread. + *

+ * This method is thread safe. + * + * @param object object to serialize + * @return serialized value of null if no serializer is available + */ + @Blocking + public static SerializedVariable.@Nullable Value serialize(@Nullable Object object) { + if (object == null) return null; - - assert !serializer.mustSyncDeserialization() || Bukkit.isPrimaryThread(); - - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - YggdrasilOutputStream yggdrasilOutputStream = Variables.yggdrasil.newOutputStream(byteOutputStream); - - yggdrasilOutputStream.writeObject(object); - yggdrasilOutputStream.flush(); - yggdrasilOutputStream.close(); - - byte[] byteArray = byteOutputStream.toByteArray(); - byte[] start = getYggdrasilStart(classInfo); - for (int i = 0; i < start.length; i++) - assert byteArray[i] == start[i] : object + " (" + classInfo.getC().getName() + "); " + Arrays.toString(start) + ", " + Arrays.toString(byteArray); - byte[] byteArrayCopy = new byte[byteArray.length - start.length]; - System.arraycopy(byteArray, start.length, byteArrayCopy, 0, byteArrayCopy.length); - - Object deserialized; - assert equals(object, - deserialized = deserialize(classInfo, new ByteArrayInputStream(byteArrayCopy))) - : object + " (" + object.getClass() + ") != " + deserialized + " (" - + (deserialized == null ? null : deserialized.getClass()) + "): " + Arrays.toString(byteArray); - - return new SerializedVariable.Value(classInfo.getCodeName(), byteArrayCopy); - } catch (IOException ex) { // shouldn't happen - Skript.exception(ex); + var result = serialize(Map.of("object", object)); + if (result == null) return null; - } + var iterator = result.iterator(); + if (!iterator.hasNext()) + return null; + return iterator.next().value(); } - private static boolean equals(final @Nullable Object o, final @Nullable Object d) { - if (o instanceof Chunk) { // CraftChunk does neither override equals nor is it a "coordinate-specific singleton" like Block - if (!(d instanceof Chunk)) - return false; - final Chunk c1 = (Chunk) o, c2 = (Chunk) d; - return c1.getWorld().equals(c2.getWorld()) && c1.getX() == c2.getX() && c1.getZ() == c2.getZ(); - } - return o == null ? d == null : o.equals(d); + /** + * The serialization process for a single object. + *

+ * This method must be called from the main thread if the serializer for + * given class info must be synchronized. + * + * @see #serialize(Object) + * @see #serialize(Map) + */ + private static SerializedVariable.Value serialize(Object object, ClassInfo classInfo) throws IOException { + assert classInfo.getSerializer() != null; + assert !classInfo.getSerializer().mustSyncDeserialization() || Bukkit.isPrimaryThread(); + + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); + YggdrasilOutputStream yggdrasilOutputStream = Variables.yggdrasil.newOutputStream(byteOutputStream); + + yggdrasilOutputStream.writeObject(object); + yggdrasilOutputStream.flush(); + yggdrasilOutputStream.close(); + + byte[] byteArray = byteOutputStream.toByteArray(); + byte[] start = getYggdrasilStart(classInfo); + for (int i = 0; i < start.length; i++) + assert byteArray[i] == start[i] : object + " (" + classInfo.getC().getName() + "); " + Arrays.toString(start) + ", " + Arrays.toString(byteArray); + byte[] byteArrayCopy = new byte[byteArray.length - start.length]; + System.arraycopy(byteArray, start.length, byteArrayCopy, 0, byteArrayCopy.length); + return new SerializedVariable.Value(classInfo.getCodeName(), byteArrayCopy); } - @Nullable - public static Object deserialize(final ClassInfo type, final byte[] value) { - return deserialize(type, new ByteArrayInputStream(value)); + /** + * Deserializes the provided set of serialized variables. + *

+ * Is blocking if the serializer for some of the variables needs to be synchronized and + * the method is not called from the main thread. + *

+ * This does skip empty variables and does not map them in the returned map to null values. + *

+ * This method is thread safe. + * + * @param variables variables to deserialize + * @return deserialized variables, returns null if the deserialization failed + * because Skript is disabled, some of the variables need to be deserialization + * on the main thread and this method was called off the main thread. + */ + @Blocking + public static @Nullable Map deserialize(Set variables) { + Map collected = new ConcurrentHashMap<>(); + Set needsSync = ConcurrentHashMap.newKeySet(); + + variables.stream().parallel().forEach(var -> { + String key = var.name(); + SerializedVariable.Value value = var.value(); + + if (value == null) + return; + + ClassInfo classInfo = getClassInfoNoError(value.type()); + if (classInfo == null) { + collected.put(key, null); + return; + } + + Serializer serializer = classInfo.getSerializer(); + if (serializer == null) { + collected.put(key, null); + return; + } + + if (serializer.mustSyncDeserialization()) { + needsSync.add(var); + return; + } + + collected.put(key, deserialize(new ByteArrayInputStream(value.data()), classInfo)); + }); + + Runnable syncDeserialization = () -> needsSync.forEach(var -> { + String key = var.name(); + SerializedVariable.Value value = var.value(); + assert value != null; + ClassInfo classInfo = getClassInfoNoError(value.type()); + var deserialized = deserialize(new ByteArrayInputStream(value.data()), classInfo); + collected.put(key, deserialized); + }); + + if (needsSync.isEmpty()) + return collected; + + if (Bukkit.isPrimaryThread()) { + syncDeserialization.run(); + } else { + if (!Skript.getInstance().isEnabled()) + // At this point we can not deserialize variables synchronously, + // we fail rather than provide partial result + return null; + try { + CompletableFuture.supplyAsync(() -> { + syncDeserialization.run(); + return null; + }, Bukkit.getScheduler().getMainThreadExecutor(Skript.getInstance())).get(); + } catch (Exception exception) { + Skript.exception(exception, "Failed to serialize variables on the main thread"); + } + } + + return collected; } - @Nullable - public static Object deserialize(final String type, final byte[] value) { - final ClassInfo ci = getClassInfoNoError(type); - if (ci == null) + /** + * Deserializes the provided variable value to an object. + *

+ * Is blocking if the serializer for the action needs to be synchronized and + * the method is not called from the main thread. + *

+ * This method is thread safe. + * + * @param value value to deserialize + * @return deserialized object of null if no serializer is available + */ + @Blocking + public static @Nullable Object deserialize(SerializedVariable. @Nullable Value value) { + if (value == null) + return null; + var result = deserialize(Set.of(new SerializedVariable("", value))); + if (result == null || result.isEmpty()) return null; - return deserialize(ci, new ByteArrayInputStream(value)); + return result.values().iterator().next(); } - @Nullable - public static Object deserialize(final ClassInfo type, InputStream value) { - Serializer s; - assert (s = type.getSerializer()) != null && (s.mustSyncDeserialization() ? Bukkit.isPrimaryThread() : true) : type + "; " + s + "; " + Bukkit.isPrimaryThread(); - YggdrasilInputStream in = null; - try { - value = new SequenceInputStream(new ByteArrayInputStream(getYggdrasilStart(type)), value); - in = Variables.yggdrasil.newInputStream(value); + /** + * Deserializes the provided variable value to an object. + *

+ * Is blocking if the serializer for the action needs to be synchronized and + * the method is not called from the main thread. + *

+ * This method is thread safe. + * + * @param value value to deserialize + * @param type type of the value + * @return deserialized object of null if no serializer is available + */ + @Blocking + public static @Nullable Object deserialize(byte[] value, ClassInfo type) { + return deserialize(value, type.getCodeName()); + } + + /** + * Deserializes the provided variable value to an object. + *

+ * Is blocking if the serializer for the action needs to be synchronized and + * the method is not called from the main thread. + *

+ * This method is thread safe. + * + * @param value value to deserialize + * @param type type of the value + * @return deserialized object of null if no serializer is available + */ + @Blocking + public static @Nullable Object deserialize(byte[] value, String type) { + return deserialize(new SerializedVariable.Value(type, value)); + } + + /** + * The deserialization process for a single object. + *

+ * This method must be called from the main thread if the serializer for + * given class info must be synchronized. + * + * @see #deserialize(byte[], ClassInfo) + * @see #deserialize(Set) + */ + private static @Nullable Object deserialize(InputStream inputStream, ClassInfo classInfo) { + assert classInfo.getSerializer() != null; + assert !classInfo.getSerializer().mustSyncDeserialization() || Bukkit.isPrimaryThread(); + + try (var sis = new SequenceInputStream(new ByteArrayInputStream(getYggdrasilStart(classInfo)), inputStream); + var in = Variables.yggdrasil.newInputStream(sis)) { return in.readObject(); - } catch (final IOException e) { // i.e. invalid save + } catch (IOException exception) { if (Skript.testing()) - e.printStackTrace(); + Skript.exception(exception, "Failed to deserialize variable of type " + classInfo.getCodeName()); return null; - } finally { - if (in != null) { - try { - in.close(); - } catch (final IOException e) {} - } - try { - value.close(); - } catch (final IOException e) {} } } /** - * Deserialises an object. + * Deserializes an object. *

- * This method must only be called from Bukkits main thread! + * This method must only be called from Bukkit main thread! * - * @param type - * @param value - * @return Deserialised value or null if the input is invalid + * @param type type of the value + * @param value value as a string + * @return deserialized value or null if the input is invalid + * @deprecated for legacy deserialization, use {@link #deserialize(byte[], String)} */ + @SuppressWarnings("removal") @Deprecated(since = "2.3.0", forRemoval = true) - @Nullable - public static Object deserialize(final String type, final String value) { + public static @Nullable Object deserialize(final String type, final String value) { assert Bukkit.isPrimaryThread(); final ClassInfo ci = getClassInfoNoError(type); if (ci == null) diff --git a/src/main/java/ch/njol/skript/sections/SecFilter.java b/src/main/java/ch/njol/skript/sections/SecFilter.java index d7aba1f51f1..5d8cd61cd24 100644 --- a/src/main/java/ch/njol/skript/sections/SecFilter.java +++ b/src/main/java/ch/njol/skript/sections/SecFilter.java @@ -10,17 +10,11 @@ import ch.njol.skript.doc.Name; import ch.njol.skript.doc.Since; import ch.njol.skript.expressions.ExprInput; -import ch.njol.skript.lang.Condition; -import ch.njol.skript.lang.Expression; -import ch.njol.skript.lang.InputSource; -import ch.njol.skript.lang.Section; +import ch.njol.skript.lang.*; import ch.njol.skript.lang.SkriptParser.ParseResult; -import ch.njol.skript.lang.TriggerItem; -import ch.njol.skript.lang.Variable; import ch.njol.skript.lang.parser.ParserInstance; import ch.njol.skript.variables.Variables; import ch.njol.util.Kleenean; -import ch.njol.util.Pair; import ch.njol.util.StringUtils; import org.bukkit.event.Event; import org.jetbrains.annotations.Nullable; @@ -126,7 +120,7 @@ public boolean init(Expression[] expressions, int matchedPattern, Kleenean is int initialSize = rawVariable.size(); // we save both because we don't yet know which will be cheaper to use. - List> toKeep = new ArrayList<>(); + List> toKeep = new ArrayList<>(); List toRemove = new ArrayList<>(); var variableIterator = Variables.getVariableIterator(varName, local, event); @@ -157,7 +151,7 @@ public boolean init(Expression[] expressions, int matchedPattern, Kleenean is // for instances where only a handful of values are removed from a large list, this can be a 400% speedup if (toKeep.size() < initialSize / 2) { Variables.deleteVariable(varName, event, local); - for (Pair pair : toKeep) + for (KeyedValue pair : toKeep) Variables.setVariable(varSubName + pair.getKey(), pair.getValue(), event, local); } else { for (String index : toRemove) diff --git a/src/main/java/ch/njol/skript/util/AsyncEffect.java b/src/main/java/ch/njol/skript/util/AsyncEffect.java index a215f21a7ea..d941b3e3cdc 100644 --- a/src/main/java/ch/njol/skript/util/AsyncEffect.java +++ b/src/main/java/ch/njol/skript/util/AsyncEffect.java @@ -1,5 +1,6 @@ package ch.njol.skript.util; +import ch.njol.skript.variables.VariablesMap; import org.bukkit.Bukkit; import org.bukkit.event.Event; import org.jetbrains.annotations.Nullable; @@ -28,7 +29,7 @@ public abstract class AsyncEffect extends Effect { protected TriggerItem walk(Event e) { debug(e, true); - Object localVars = Variables.removeLocals(e); // Back up local variables + VariablesMap localVars = Variables.removeLocals(e); // Back up local variables if (!Skript.getInstance().isEnabled()) // See https://github.com/SkriptLang/Skript/issues/3702 return null; diff --git a/src/main/java/ch/njol/skript/variables/FlatFileStorage.java b/src/main/java/ch/njol/skript/variables/FlatFileStorage.java index bc76ab838eb..a894e4b701f 100644 --- a/src/main/java/ch/njol/skript/variables/FlatFileStorage.java +++ b/src/main/java/ch/njol/skript/variables/FlatFileStorage.java @@ -2,30 +2,30 @@ import ch.njol.skript.Skript; import ch.njol.skript.config.SectionNode; -import ch.njol.skript.lang.Variable; import ch.njol.skript.log.SkriptLogger; import ch.njol.skript.registrations.Classes; import ch.njol.skript.util.ExceptionUtils; import ch.njol.skript.util.FileUtils; import ch.njol.skript.util.Task; -import ch.njol.skript.util.Utils; -import ch.njol.skript.util.Version; -import ch.njol.util.NotifyingReference; import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; import java.io.BufferedReader; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Map.Entry; -import java.util.TreeMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,13 +34,7 @@ * A variable storage that stores its content in a * comma-separated value file (CSV file). */ -/* - * TODO use a database (SQLite) instead and only load a limited amount of variables into RAM - e.g. 2 GB (configurable). - * If more variables are available they will be loaded when - * accessed. (rem: print a warning when Skript starts) - * rem: store null variables (in memory) to prevent looking up the same variables over and over again - */ -public class FlatFileStorage extends VariablesStorage { +public class FlatFileStorage extends VariableStorage { /** * The {@link Charset} used in the CSV storage file. @@ -50,79 +44,70 @@ public class FlatFileStorage extends VariablesStorage { /** * The delay for the save task. */ - private static final long SAVE_TASK_DELAY = 5 * 60 * 20; + // TODO move to database configuration + private static final long SAVE_TASK_DELAY = 5 * 60 * 20; // 5 minutes /** * The period for the save task, how long (in ticks) between each save. */ - private static final long SAVE_TASK_PERIOD = 5 * 60 * 20; + // TODO move to database configuration + private static final long SAVE_TASK_PERIOD = 5 * 60 * 20; // 5 minutes /** - * A reference to the {@link PrintWriter} that is used to write - * to the {@link #file}. - *

- * A Lock on this object must be acquired after connectionLock - * if that lock is used - * (and thus also after {@link Variables#getReadLock()}). + * The amount of variable changes needed to save the variables into a file. */ - private final NotifyingReference changesWriter = new NotifyingReference<>(); + // TODO move to database configuration + private static int REQUIRED_CHANGES_FOR_RESAVE = 1000; /** - * Whether the storage has been loaded. + * The amount of variable changes written since the last full save. + * + * @see #REQUIRED_CHANGES_FOR_RESAVE */ - private volatile boolean loaded = false; + private final AtomicInteger changes = new AtomicInteger(0); /** - * The amount of {@link #changes} needed - * for a new {@link #saveVariables(boolean) save}. + * Whether the storage is being saved now (written to a file). */ - private static int REQUIRED_CHANGES_FOR_RESAVE = 1000; + private final AtomicBoolean isSaving = new AtomicBoolean(false); /** - * The amount of variable changes written since the last full save. - * - * @see #REQUIRED_CHANGES_FOR_RESAVE + * Variables map of variables managed by this storage. */ - private final AtomicInteger changes = new AtomicInteger(0); + private final VariablesMap variablesMap = new VariablesMap(); /** - * The save task. - * - * @see #changes - * @see #saveVariables(boolean) - * @see #REQUIRED_CHANGES_FOR_RESAVE - * @see #SAVE_TASK_DELAY - * @see #SAVE_TASK_PERIOD + * Executor used for scheduling the storage save. */ - @Nullable - private Task saveTask; + private final ExecutorService saveExecutor; /** - * Whether there was an error while loading variables. - *

- * Set back to {@code false} when a backup has been made - * of the variable file that caused the error. + * Task for saving variables into the file. + */ + private @Nullable Task saveTask; + + /** + * Whether the storage has been closed. */ - private boolean loadError = false; + private final AtomicBoolean closed = new AtomicBoolean(false); /** * Create a new CSV storage of the given name. * - * @param type the databse type i.e. CSV. + * @param source the source of this storage. + * @param type the database type i.e. CSV. */ - FlatFileStorage(String type) { - super(type); + public FlatFileStorage(SkriptAddon source, String type) { + super(source, type); + saveExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "FlatFileStorage-Variable-Save-" + source.name() + "-" + type); + thread.setDaemon(false); // finish save on shutdown + return thread; + }); } - /** - * Loads the variables in the CSV file. - *

- * Doesn't lock the connection, as required by - * {@link Variables#variableLoaded(String, Object, VariablesStorage)}. - */ - @SuppressWarnings("deprecation") @Override - protected boolean load_i(SectionNode sectionNode) { + protected final boolean load(SectionNode sectionNode) { SkriptLogger.setNode(null); if (file == null) { @@ -130,22 +115,9 @@ protected boolean load_i(SectionNode sectionNode) { return false; } - // Keep track of loading errors - IOException ioException = null; - int unsuccessfulVariableCount = 0; - StringBuilder invalid = new StringBuilder(); + Set collected = new HashSet<>(); - // The Skript version this CSV was created with - Version csvSkriptVersion; - - // Some variables used to allow legacy CSV files to be loaded - Version v2_0_beta3 = new Version(2, 0, "beta 3"); - boolean update2_0_beta3 = false; - Version v2_1 = new Version(2, 1); - boolean update2_1 = false; - - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(Files.newInputStream(file.toPath()), FILE_CHARSET))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(file.toPath()), FILE_CHARSET))) { String line; int lineNum = 0; while ((line = reader.readLine()) != null) { @@ -153,124 +125,110 @@ protected boolean load_i(SectionNode sectionNode) { line = line.trim(); - if (line.isEmpty() || line.startsWith("#")) { - // Line doesn't contain variable - if (line.startsWith("# version:")) { - // Update the version accordingly - - try { - csvSkriptVersion = new Version(line.substring("# version:".length()).trim()); - update2_0_beta3 = csvSkriptVersion.isSmallerThan(v2_0_beta3); - update2_1 = csvSkriptVersion.isSmallerThan(v2_1); - } catch (IllegalArgumentException ignored) { - } - } - + if (line.isEmpty() || line.startsWith("#")) continue; - } String[] split = splitCSV(line); if (split == null || split.length != 3) { - // Invalid CSV line - + // invalid CSV line Skript.error("invalid amount of commas in line " + lineNum + " ('" + line + "')"); - if (invalid.length() != 0) - invalid.append(", "); - - invalid.append(split == null ? "" : split[0]); - unsuccessfulVariableCount++; continue; } - if (split[1].equals("null")) { - Variables.variableLoaded(split[0], null, this); + String key = split[0]; + String type = split[1]; + SerializedVariable serializedVariable; + if (type.equals("null")) { + serializedVariable = new SerializedVariable(key, null); } else { - Object deserializedValue; - if (update2_1) { - // Use old deserialization if variables come from old Skript version - deserializedValue = Classes.deserialize(split[1], split[2]); - } else { - deserializedValue = Classes.deserialize(split[1], decode(split[2])); - } - - if (deserializedValue == null) { - // Couldn't deserialize variable - if (invalid.length() != 0) - invalid.append(", "); - - invalid.append(split[0]); - unsuccessfulVariableCount++; - continue; - } - - // Legacy - if (deserializedValue instanceof String && update2_0_beta3) { - deserializedValue = Utils.replaceChatStyles((String) deserializedValue); - } - - Variables.variableLoaded(split[0], deserializedValue, this); + serializedVariable = new SerializedVariable(key, type, decode(split[2])); } + collected.add(serializedVariable); } } catch (IOException e) { - loadError = true; - ioException = e; + Skript.exception(e, "Failed to load variables from storage"); + return false; } - if (ioException != null || unsuccessfulVariableCount > 0 || update2_1) { - // Something's wrong (or just an old version) - if (unsuccessfulVariableCount > 0) { - Skript.error(unsuccessfulVariableCount + " variable" + (unsuccessfulVariableCount == 1 ? "" : "s") + - " could not be loaded!"); - Skript.error("Affected variables: " + invalid.toString()); - } - - if (ioException != null) { - Skript.error("An I/O error occurred while loading the variables: " + ExceptionUtils.toString(ioException)); - Skript.error("This means that some to all variables could not be loaded!"); - } - - try { - if (update2_1) { - Skript.info("[2.1] updating " + file.getName() + " to the new format..."); - } - - // Back up CSV file - File backupFile = FileUtils.backup(file); - Skript.info("Created a backup of " + file.getName() + " as " + backupFile.getName()); + // TODO conversions from v2_0_beta3 and v2_1 + // do we really need this? those versions are from 2017 - loadError = false; - } catch (IOException ex) { - Skript.error("Could not backup " + file.getName() + ": " + ex.getMessage()); - } - } + // TODO logging about failed deserialization - if (update2_1) { - // Save variables in new format - saveVariables(false); - Skript.info(file.getName() + " successfully updated."); - } + // TODO what about variables that do not match pattern? - connect(); + var deserialized = Classes.deserialize(collected); + assert deserialized != null; + deserialized.forEach(variablesMap::setVariable); - // Start the save task saveTask = new Task(Skript.getInstance(), SAVE_TASK_DELAY, SAVE_TASK_PERIOD, true) { @Override public void run() { - // Due to concurrency, the amount of changes may change between the get and set call - // but that's not a big issue - if (changes.get() >= REQUIRED_CHANGES_FOR_RESAVE) { - saveVariables(false); - changes.set(0); - } + if (changes.get() > 0) + saveAsync(); } }; - return ioException == null; + return true; } - @Override - protected void allLoaded() { - // no transaction support + /** + * Calls the save executor to perform the rewrite of the CSV file. + */ + private void saveAsync() { + if (closed.get()) + return; + if (isSaving.compareAndSet(false, true)) { + saveExecutor.execute(() -> { + try { + performSave(variablesMap.getAll()); + } finally { + isSaving.set(false); + } + }); + } + } + + /** + * Completely rewrites the CSV file. + */ + private void performSave(Map snapshot) { + assert file != null; + File tempFile = new File(file.getParentFile(), file.getName() + ".temp"); + + Set serializedVariables = Classes.serialize(snapshot); + if (serializedVariables == null) { + if (Skript.debug()) { + Skript.warning("Failed to save the variables off main thread, this may happen when Skript gets disabled."); + Skript.warning("No data is lost, final save will run synchronously on the main thread."); + } + return; + } + + try (PrintWriter pw = new PrintWriter(tempFile, FILE_CHARSET)) { + pw.println("# === Skript's variable storage ==="); + pw.println("# Please do not modify this file manually!"); + pw.println("#"); + pw.println("# version: " + Skript.getVersion()); + pw.println(); + + serializedVariables.forEach(variable -> { + if (variable.value() == null) + return; + String name = variable.name(); + String type = variable.value().type(); + String encoded = encode(variable.value().data()); + writeCSV(pw, name, type, encoded); + }); + + pw.println(); + pw.flush(); + pw.close(); + FileUtils.move(tempFile, file, true); + } catch (IOException e) { + Skript.error("Unable to make a save of the database '" + getUserConfigurationName() + + "' (no variables are lost): " + ExceptionUtils.toString(e)); + } } @Override @@ -284,222 +242,53 @@ protected File getFile(String fileName) { } @Override - protected final void disconnect() { - synchronized (connectionLock) { - clearChangesQueue(); - synchronized (changesWriter) { - PrintWriter printWriter = changesWriter.get(); - - if (printWriter != null) { - printWriter.close(); - changesWriter.set(null); - } - } - } + public @Nullable Object getVariable(String name) { + return variablesMap.getVariable(name); } @Override - protected final boolean connect() { - synchronized (connectionLock) { - synchronized (changesWriter) { - assert file != null; // file should be non-null after load - - if (changesWriter.get() != null) - return true; - - // Open the file stream, and create the PrintWriter with it - try (FileOutputStream fos = new FileOutputStream(file, true)) { - changesWriter.set(new PrintWriter(new OutputStreamWriter(fos, FILE_CHARSET))); - loaded = true; - return true; - } catch (IOException e) { // close() might throw ANY IOException - //noinspection ThrowableNotThrown - Skript.exception(e); - return false; - } - } + public void setVariable(String name, @Nullable Object value) { + variablesMap.setVariable(name, value); + int currentChanges = changes.incrementAndGet(); + if (currentChanges >= REQUIRED_CHANGES_FOR_RESAVE) { + saveAsync(); } } @Override - public void close() { - clearChangesQueue(); - super.close(); - saveVariables(true); // also closes the writer + public long loadedVariables() { + return variablesMap.size(); } @Override - protected boolean save(String name, @Nullable String type, @Nullable byte[] value) { - synchronized (connectionLock) { - synchronized (changesWriter) { - if (!loaded && type == null) { - // deleting variables is not really required for this kind of storage, - // as it will be completely rewritten every once in a while, - // and at least once when the server stops. - return true; - } - - // Get the PrintWriter, waiting for it to be available if needed - PrintWriter printWriter; - while ((printWriter = changesWriter.get()) == null) { - try { - changesWriter.wait(); - } catch (InterruptedException e) { - // Re-interrupt thread - Thread.currentThread().interrupt(); - } - } - - writeCSV(printWriter, name, type, value == null ? "" : encode(value)); - printWriter.flush(); - - changes.incrementAndGet(); - } - } - return true; - } - - /** - * Completely rewrites the CSV file. - *

- * The {@code finalSave} argument is used to determine if - * the {@link #saveTask save} and {@link #backupTask backup} tasks - * should be cancelled, and if the storage should reconnect after saving. - * - * @param finalSave whether this is the last save in this session or not. - */ - public final void saveVariables(boolean finalSave) { - if (finalSave) { - // Cancel save and backup tasks, not needed with final save anyway - if (saveTask != null) - saveTask.cancel(); - if (backupTask != null) - backupTask.cancel(); + public void close() { + if (!closed.compareAndSet(false, true)) + return; + if (saveTask != null) { + saveTask.cancel(); + saveTask = null; } - + // it can not finish the save anyway because Skript is disabled and + // serialization will fail off main thread as it can not schedule + // tasks to serialize such variables. + // we can shutdown now as all variables are on heap and will be + // saved once again on the main thread + saveExecutor.shutdownNow(); + + // wait for the background thread to actually release the file try { - // Acquire read lock - Variables.getReadLock().lock(); - - synchronized (connectionLock) { - try { - if (file == null) { - // This storage requires a file, so file should be nonnull - assert false : this; - return; - } - - disconnect(); - - if (loadError) { - // There was an error while loading the CSV file, create a backup of it - try { - File backup = FileUtils.backup(file); - Skript.info("Created a backup of the old " + file.getName() + " as " + backup.getName()); - loadError = false; - } catch (IOException e) { - Skript.error("Could not backup the old " + file.getName() + ": " + ExceptionUtils.toString(e)); - Skript.error("No variables are saved!"); - return; - } - } - - // Write the variables to a temporary file, giving less problems if saving fails - // (if saving fails during writing to the actual file, - // the data in the actual file may be partially lost) - File tempFile = new File(file.getParentFile(), file.getName() + ".temp"); - - try (PrintWriter pw = new PrintWriter(tempFile, "UTF-8")) { - pw.println("# === Skript's variable storage ==="); - pw.println("# Please do not modify this file manually!"); - pw.println("#"); - pw.println("# version: " + Skript.getVersion()); - pw.println(); - save(pw, "", Variables.getVariables()); - pw.println(); - pw.flush(); - pw.close(); - FileUtils.move(tempFile, file, true); - } catch (IOException e) { - Skript.error("Unable to make a final save of the database '" + getUserConfigurationName() + - "' (no variables are lost): " + ExceptionUtils.toString(e)); - // FIXME happens at random - check locks/threads - } - } finally { - // Reconnect if needed - if (!finalSave) { - connect(); - } - } - } - } finally { - Variables.getReadLock().unlock(); - boolean gotWriteLock = Variables.variablesLock.writeLock().tryLock(); - if (gotWriteLock) { // Only process queue now if it doesn't require us to wait - try { - Variables.processChangeQueue(); - } finally { - Variables.variablesLock.writeLock().unlock(); - } + if (!saveExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + Skript.warning("Variable save thread took too long to shutdown. Final save might fail."); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } - } - - /** - * Saves the variables. - *

- * This method uses the sorted variables map to save the variables in order. - * - * @param pw the print writer to write the CSV lines too. - * @param parent The parent's name with {@link Variable#SEPARATOR} at the end. - * @param map the variables map. - */ - @SuppressWarnings("unchecked") - private void save(PrintWriter pw, String parent, TreeMap map) { - if (parent.startsWith(Variable.EPHEMERAL_VARIABLE_TOKEN)) - // Skip ephemeral variables - return; - // Iterate over all children - for (Entry childEntry : map.entrySet()) { - Object childNode = childEntry.getValue(); - String childKey = childEntry.getKey(); - - if (childNode == null) - continue; // Leaf node - - if (childNode instanceof TreeMap) { - // TreeMap found, recurse - save(pw, parent + childKey + Variable.SEPARATOR, (TreeMap) childNode); - } else { - // Remove variable separator if needed - String name = childKey == null ? parent.substring(0, parent.length() - Variable.SEPARATOR.length()) : parent + childKey; - - if (name.startsWith(Variable.EPHEMERAL_VARIABLE_TOKEN)) - // Skip ephemeral variables - continue; - - try { - // Loop over storages to make sure this variable is ours to store - for (VariablesStorage storage : Variables.STORAGES) { - if (storage.accept(name)) { - if (storage == this) { - // Serialize the value - SerializedVariable.Value serializedValue = Classes.serialize(childNode); - - // Write the CSV line - if (serializedValue != null) - writeCSV(pw, name, serializedValue.type, encode(serializedValue.data)); - } - - break; - } - } - } catch (Exception ex) { - //noinspection ThrowableNotThrown - Skript.exception(ex, "Error saving variable named " + name); - } - } + // now write to file.temp + if (changes.get() > 0) { + Map snapshot = variablesMap.getAll(); + changes.set(0); + performSave(snapshot); } } @@ -558,8 +347,7 @@ static byte[] decode(String hex) { * * @see #CSV_LINE_PATTERN */ - @Nullable - static String[] splitCSV(String line) { + static String @Nullable [] splitCSV(String line) { Matcher matcher = CSV_LINE_PATTERN.matcher(line); int lastEnd = 0; @@ -627,7 +415,6 @@ private static void writeCSV(PrintWriter printWriter, String... values) { /** * Change the required amount of variable changes until variables are saved. * Cannot be zero or less. - * @param value */ public static void setRequiredChangesForResave(int value) { if (value <= 0) { diff --git a/src/main/java/ch/njol/skript/variables/JdbcStorage.java b/src/main/java/ch/njol/skript/variables/JdbcStorage.java new file mode 100644 index 00000000000..be6275a2129 --- /dev/null +++ b/src/main/java/ch/njol/skript/variables/JdbcStorage.java @@ -0,0 +1,713 @@ +package ch.njol.skript.variables; + +import ch.njol.skript.Skript; +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.lang.Variable; +import ch.njol.skript.log.SkriptLogger; +import ch.njol.skript.registrations.Classes; +import ch.njol.skript.util.Task; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.jetbrains.annotations.Blocking; +import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.sql.*; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.StampedLock; + +/** + * Storage for Skript variables that uses SQL database. + *

+ * This class is abstract and should be extended to implement specific SQL database storage. + *

+ * This implementation does not synchronize the variables loaded on server with variables + * from the connected database; it does not update with each transaction. It is efficient + * local alternative for implementations such as {@link FlatFileStorage}. + *

+ * The default implementation is SQLite/Postgres syntax, but implementations are expected + * to override methods for supplying queries: + *

    + *
  • {@link #createTableQuery()}
    • + *
  • {@link #readSingleQuery(Connection)}
    • + *
  • {@link #readListQuery(Connection)}
    • + *
  • {@link #writeSingleQuery(Connection)}
    • + *
  • {@link #writeMultipleQuery(Connection)}
    • + *
  • {@link #deleteSingleQuery(Connection)}
    • + *
  • {@link #deleteListQuery(Connection)}
    • + *
+ */ +public abstract class JdbcStorage extends VariableStorage { + + protected static final String DEFAULT_TABLE_NAME = "variables21"; + + public static final int MAX_VARIABLE_NAME_LENGTH = 380; // MySQL: 767 bytes max; cannot set max bytes, only max characters + public static final int MAX_CLASS_CODENAME_LENGTH = 50; // checked when registering a class + public static final int MAX_VALUE_SIZE = 10000; + + /** + * The delay for the save task. + */ + // TODO move to database configuration (this could be shared factory method in VariableStorage, as + // FlatFileStorage should also have those options) + private static final long SAVE_TASK_DELAY = 5 * 60 * 20; // 5 minutes + + /** + * The period for the save task, how long (in ticks) between each save. + */ + // TODO move to database configuration + private static final long SAVE_TASK_PERIOD = 5 * 60 * 20; // 5 minutes + + /** + * The amount of variable changes needed to save the variables into a file. + */ + // TODO move to database configuration + private static final int REQUIRED_CHANGES_FOR_RESAVE = 1000; + + /** + * Name of the table where variables are being saved. + */ + // TODO move to database configuration, needs sanitization checks + protected final String table = DEFAULT_TABLE_NAME; + + /** + * Database source. + */ + protected @Nullable HikariDataSource database; + + /** + * The amount of variable changes written since the last full save. + * + * @see #REQUIRED_CHANGES_FOR_RESAVE + */ + private final AtomicInteger changes = new AtomicInteger(0); + + /** + * Whether the storage is being saved now (written to a file). + */ + private final AtomicBoolean isSaving = new AtomicBoolean(false); + + /** + * Variables currently loaded in memory. + *

+ * This map contains currently loaded variables by this storage. + * Once variable is loaded (either from database or set), it stays in this + * map until the storage is closed. + */ + private final VariablesMap variablesMap = new VariablesMap(); + + /** + * Variables that have been modified since the last save (write buffer). + */ + private volatile VariablesMap dirty = new VariablesMap(); + + /** + * Variables/Branches that have been deleted since the last save. + *

+ * All objects in this map are of type {@link Marker}. + */ + private volatile VariablesMap cleared = new VariablesMap(); + + /** + * Variables/Branches that have been loaded from the database to + * the {@link #variablesMap}. + *

+ * All objects in this map are of type {@link Marker}. + */ + private final VariablesMap loaded = new VariablesMap(); + + /** + * Represents a marker in a variables map. + */ + private static final class Marker { + + /** + * Whether this marker applies to the single variable value, e.g.: ({@code {this::node}}). + */ + volatile boolean single; + + /** + * Whether this marker applies to the variable children ({@code {this::node::*}}). + */ + volatile boolean branch; + + Marker() { + this(false, false); + } + + Marker(boolean single, boolean branch) { + this.single = single; + this.branch = branch; + } + + } + + /** + * Executor used for scheduling the storage save. + */ + private final ExecutorService saveExecutor; + + /** + * Task for saving variables into the file. + */ + private @Nullable Task saveTask; + + /** + * Whether the storage has been closed. + */ + private final AtomicBoolean closed = new AtomicBoolean(false); + + /** + * Lock for synchronization of writing loaded variables into + * the database and disposing them to free heap. + */ + private final StampedLock lock = new StampedLock(); + + protected JdbcStorage(SkriptAddon source, String type) { + super(source, type); + saveExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "JdbcStorage-Variable-Save-" + source.name() + "-" + type); + thread.setDaemon(false); // finish save on shutdown + return thread; + }); + } + + /** + * Build a HikariConfig from the Skript config.sk SectionNode of this database. + * + * @param sectionNode The configuration section from the config.sk that defines this database. + * @return A HikariConfig implementation. Or null if failure. + */ + protected abstract @Nullable HikariConfig configuration(SectionNode sectionNode); + /** + * @return SQL query to create the variables table if it does not exist + *
Required Columns: + *

    + *
  • name: Primary Key (Varchar/Text)
    • + *
  • type: The serialization type (Varchar/Text)
    • + *
  • value: The binary data (Blob)
    • + *
+ */ + // language=SQL + protected String createTableQuery() { + return "CREATE TABLE IF NOT EXISTS " + table + " (" + + "name VARCHAR(" + MAX_VARIABLE_NAME_LENGTH + ") PRIMARY KEY," + + "type VARCHAR(" + MAX_CLASS_CODENAME_LENGTH + ")," + + "value BLOB(" + MAX_VALUE_SIZE + ")" + + ");"; + } + + /** + * @param connection connnection + * @return The SQL query to select a single variable's type and value by its name. + *
Expected Params: name (String) + */ + protected PreparedStatement readSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("SELECT type, value FROM " + table + " WHERE name = ?"); + } + + /** + * @param connection connnection + * @return The SQL query to select all variables (name, type, value) that start with a specific prefix. + *
Expected Params: name_prefix% (String) - usually used with LIKE + */ + protected PreparedStatement readListQuery(Connection connection) throws SQLException { + return connection.prepareStatement("SELECT name, type, value FROM " + table + " WHERE name LIKE ?"); + } + + /** + * @param connection connnection + * @return The SQL query to insert or update (upsert) a single variable. + *
Expected Params: name, type, value + */ + protected PreparedStatement writeSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("INSERT INTO " + table + " (name, type, value) VALUES (?, ?, ?) " + + "ON CONFLICT(name) DO UPDATE SET type=excluded.type, value=excluded.value"); + } + + /** + * @param connection connnection + * @return The SQL query used for JDBC batch writes. + *
Usually identical to {@link #writeSingleQuery(Connection)} + *
Expected Params: name, type, value + */ + protected PreparedStatement writeMultipleQuery(Connection connection) throws SQLException { + return writeSingleQuery(connection); + } + + /** + * @param connection connnection + * @return The SQL query to delete a single variable by name. + *
Expected Params: name + */ + protected PreparedStatement deleteSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("DELETE FROM " + table + " WHERE name = ?"); + } + + /** + * @param connection connnection + * @return The SQL query to delete multiple variables matching a prefix (list deletion). + *
Expected Params: name_prefix% (String) - usually used with LIKE + */ + protected PreparedStatement deleteListQuery(Connection connection) throws SQLException { + return connection.prepareStatement("DELETE FROM " + table + " WHERE name LIKE ?"); + } + + @Override + protected boolean loadAbstract(SectionNode sectionNode) { + HikariConfig configuration = configuration(sectionNode); + if (configuration == null) + return false; + + SkriptLogger.setNode(null); + + try { + database = new HikariDataSource(configuration); + } catch (Exception exception) { + Skript.error("Cannot connect to the database '" + getUserConfigurationName() + + "'! Please make sure that all settings are correct: " + exception.getLocalizedMessage()); + return false; + } + + if (database.isClosed()) { + Skript.error("Cannot connect to the database '" + getUserConfigurationName() + "'! Please make sure " + + "that all settings are correct."); + return false; + } + + // Create the table. + try { + try (Connection connection = database.getConnection()) { + Statement statement = connection.createStatement(); + //noinspection SqlSourceToSinkFlow + statement.execute(createTableQuery()); + } + } catch (SQLException e) { + Skript.error("Could not create the variables table '" + table + "' in the database '" + + getUserConfigurationName() + "': " + e.getLocalizedMessage()); + return false; + } + + saveTask = new Task(Skript.getInstance(), SAVE_TASK_DELAY, SAVE_TASK_PERIOD, true) { + @Override + public void run() { + if (changes.get() > 0) + saveAsync(); + } + }; + + return load(sectionNode); + } + + @Override + protected boolean load(SectionNode sectionNode) { + return true; + } + + @Override + public void setVariable(String name, @Nullable Object value) { + if (name.length() > MAX_VARIABLE_NAME_LENGTH) { + Skript.error("Failed to set variable '" + name + "' due to it exceeding the max name length"); + return; + } + + long stamp = lock.readLock(); + try { + // update the read variables map + variablesMap.setVariable(name, value); + // update the writes variables map + dirty.setVariable(name, value); + + // value is cleared, update the cleared variables map + if (value == null) { + boolean isList = name.endsWith(Variable.SEPARATOR + "*"); + if (isList) { + // we clear parent; we can remove information about individual child clears + cleared.setVariable(name, null); + String parent = name.substring(0, name.length() - (Variable.SEPARATOR.length() + 1)); + Marker marker = (Marker) cleared.computeIfAbsent(parent, k -> new Marker()); + marker.branch = true; + } else { + // check if parent is already cleared; if so, no need to mark individual child + String[] parts = Variables.splitVariableName(name); + StringBuilder buffer = new StringBuilder(); + boolean parentCleared = false; + for (int i = 0; i < parts.length - 1 /* we do not check self, only parents */; i++) { + if (i > 0) + buffer.append(Variable.SEPARATOR); + buffer.append(parts[i]); + var found = cleared.getVariable(buffer.toString()); + if (found instanceof Marker marker && marker.branch) { + parentCleared = true; + break; + } + } + if (!parentCleared) { // no parent cleared, we clear the single variable + Marker marker = (Marker) cleared.computeIfAbsent(name, k -> new Marker()); + marker.single = true; + } + } + } + } finally { + lock.unlockRead(stamp); + } + + if (changes.incrementAndGet() >= REQUIRED_CHANGES_FOR_RESAVE) + saveAsync(); + } + + @Override + @SuppressWarnings("OptionalAssignedToNull") + public @Nullable Object getVariable(String name) { + if (name.length() > MAX_VARIABLE_NAME_LENGTH) { + Skript.error("Failed to get variable '" + name + "' due to it exceeding the max name length"); + return null; + } + + Optional got = null; + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) + got = getLoadedVariable(name); + + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + got = getLoadedVariable(name); + } finally { + lock.unlockRead(stamp); + } + } + + if (got != null) + return got.orElse(null); + + // variable has not been loaded from the database yet + stamp = lock.writeLock(); // TODO possible improvement? we lock the world here + try { + // check if loaded during the wait for the write lock + if (hasBeenLoaded(name)) + return variablesMap.getVariable(name); + // if not then load from the database + return loadFromDatabase(name); + } finally { + lock.unlockWrite(stamp); + } + } + + /** + * Returns whether variable (single or list) was already loaded from the + * database connection in the past. + * + * @param name name of the variable + * @return whether it has already been loaded + */ + private boolean hasBeenLoaded(String name) { + boolean isList = name.endsWith(Variable.SEPARATOR + "*"); + + // single variable that is not set + if (!isList && loaded.getVariable(name) instanceof Marker marker && marker.single) + return true; + + // check if any parent list was loaded + String[] parts = Variables.splitVariableName(name); + StringBuilder buffer = new StringBuilder(); + for (int i = 0; i < parts.length - 1 /* we do not check self, only parents */; i++) { + if (i > 0) + buffer.append(Variable.SEPARATOR); + buffer.append(parts[i]); + var found = loaded.getVariable(buffer.toString()); + if (found instanceof Marker marker && marker.branch) { + return true; + } + } + + return false; + } + + /** + * Returns value of a already loaded variable. + *

+ * If the variable has been loaded from database before it will + * be returned as an optional (empty if it has no value set). + * If not {@code null} is returned. + * + * @param name name of the variable (single or list) + * @return variable value, empty if not set, {@code null} if not loaded + */ + private @Nullable Optional getLoadedVariable(String name) { + Object value = variablesMap.getVariable(name); + if (value != null) + return Optional.of(value); + //noinspection OptionalAssignedToNull + return hasBeenLoaded(name) ? Optional.empty() : null; + } + + /** + * Loads variable from a database (both single and list). + *

+ * Is blocking if the deserialization of some values must by synchronized + * on the main thread. + * + * @param name name of the variable to load + * @return its value + */ + @Blocking + private @Nullable Object loadFromDatabase(String name) { + if (database == null || database.isClosed() || closed.get()) + return null; + + boolean isList = name.endsWith(Variable.SEPARATOR + "*"); + String param = isList ? name.substring(0, name.length() - 1) + "%" : name; + + Object result = null; + + try (Connection conn = database.getConnection(); + PreparedStatement stmt = isList ? readListQuery(conn) : readSingleQuery(conn)) { + + stmt.setString(1, param); + try (ResultSet rs = stmt.executeQuery()) { + if (isList) { + Set variables = new HashSet<>(); + while (rs.next()) { + String key = rs.getString("name"); + String type = rs.getString("type"); + byte[] data = rs.getBytes("value"); + variables.add(new SerializedVariable(key, type, data)); + } + var deserialized = Classes.deserialize(variables); + if (deserialized != null) { + deserialized.forEach(variablesMap::setVariable); + result = variablesMap.getVariable(name); + } + } else { + if (rs.next()) { + String type = rs.getString("type"); + byte[] data = rs.getBytes("value"); + result = Classes.deserialize(data, type); + variablesMap.setVariable(name, result); + } + } + } + + String actualName = isList + ? name.substring(0, name.length() - (Variable.SEPARATOR.length() + 1)) + : name; + Marker marker = (Marker) loaded.computeIfAbsent(actualName, k -> new Marker()); + if (isList) { + marker.branch = true; + } else { + marker.single = true; + } + + } catch (SQLException exception) { + Skript.error("Error loading variable '" + name + "': " + exception.getLocalizedMessage()); + } + + return result; + } + + /** + * Calls the save executor to perform the rewrite of the CSV file. + */ + private void saveAsync() { + if (closed.get()) + return; + if (isSaving.compareAndSet(false, true)) { + saveExecutor.execute(() -> { + try { + performSave(); + } finally { + isSaving.set(false); + } + }); + } + } + + /** + * Writes the uncommited changes to the database. + *

+ * Is blocking if the serialization of some values must by synchronized + * on the main thread. + */ + private void performSave() { + if (changes.get() == 0 || database == null) + return; + + VariablesMap snapshotDirty; + VariablesMap snapshotCleared; + + // swap; this essentially clears the uncommited change maps + // TODO critical, this can cause data lost if the executor is shutdown on close + // as the final save will have empty maps but not everything finished saving. + // There must be a recover if the peformSave fails + long stamp = lock.writeLock(); + try { + snapshotDirty = dirty; + snapshotCleared = cleared; + + dirty = new VariablesMap(); + cleared = new VariablesMap(); + changes.set(0); + } finally { + lock.unlockWrite(stamp); + } + + if (snapshotDirty.isEmpty() && snapshotCleared.isEmpty()) + return; + + try (Connection conn = database.getConnection()) { + conn.setAutoCommit(false); + + beforeSave(conn); + + // process deletions + if (!snapshotCleared.isEmpty()) { + try (PreparedStatement deleteSingle = deleteSingleQuery(conn); + PreparedStatement deleteList = deleteListQuery(conn)) { + + Map clears = snapshotCleared.getAll(); + for (Map.Entry entry : clears.entrySet()) { + String key = entry.getKey(); + Marker marker = (Marker) entry.getValue(); + if (marker.single) { + deleteSingle.setString(1, key); + deleteSingle.addBatch(); + } + if (marker.branch) { + deleteList.setString(1, key + Variable.SEPARATOR + "%"); + deleteList.addBatch(); + } + } + deleteSingle.executeBatch(); + deleteList.executeBatch(); + } + } + + // process updates + if (!snapshotDirty.isEmpty()) { + try (PreparedStatement upsert = writeMultipleQuery(conn)) { + Map updates = snapshotDirty.getAll(); + var serialized = Classes.serialize(updates); + + if (serialized == null) { + if (Skript.debug()) { + Skript.warning("Failed to save the variables off main thread, this may happen when Skript gets disabled."); + Skript.warning("No data is lost, final save will run synchronously on the main thread."); + } + return; + } + + for (SerializedVariable variable : serialized) { + if (variable.value() == null) + continue; + + String name = variable.name(); + String type = variable.value().type(); + byte[] data = variable.value().data(); + + if (data.length > MAX_VALUE_SIZE) { + Skript.error("Failed to save variable '" + name + "' due to it exceeding the max data length"); + continue; + } + + upsert.setString(1, name); + upsert.setString(2, type); + upsert.setBytes(3, data); + upsert.addBatch(); + } + upsert.executeBatch(); + } + } + + conn.commit(); + afterSave(conn); + } catch (SQLException exception) { + Skript.error("Failed to save variables to database: " + exception.getLocalizedMessage()); + } + } + + /** + * Runs before the variable changes save. + *

+ * This is already a part of the transaction that saves the variable values + * into the database. + * + * @param conn connection + */ + protected void beforeSave(Connection conn) throws SQLException { + } + + /** + * Runs after the variables are saved into memory. + *

+ * This is after the transaction for the variable save has been committed. + * + * @param conn connection + */ + protected void afterSave(Connection conn) throws SQLException { + } + + @Override + public final void close() { + if (!closed.compareAndSet(false, true)) + return; + if (saveTask != null) { + saveTask.cancel(); + saveTask = null; + } + // it can not finish the save anyway because Skript is disabled and + // serialization will fail off main thread as it can not schedule + // tasks to serialize such variables + // we shutdown safely to avoid data corruption + saveExecutor.shutdown(); + + try { + if (!saveExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + Skript.warning("Variable save thread took too long to shutdown. Final save might fail."); + saveExecutor.shutdownNow(); + if (!saveExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + Skript.error("Variable save thread failed to shut down!"); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + saveExecutor.shutdownNow(); + } + + if (database != null) { + Skript.info("Performing final variable save for '" + getUserConfigurationName() + "'"); + performSave(); + Skript.info("Closing the database '" + getUserConfigurationName() + "'"); + try { + closeDatabase(); + } catch (SQLException exception) { + Skript.exception(exception, "Failed to close the database '" + getUserConfigurationName() + "'"); + } + } + } + + /** + * Called when closing the database after Skript shutdown. + *

+ * This method must close the database source. + */ + protected void closeDatabase() throws SQLException { + if (database != null && !database.isClosed()) { + database.close(); + } + } + + @Override + public long loadedVariables() { + return variablesMap.size(); + } + +} diff --git a/src/main/java/ch/njol/skript/variables/MySQLStorage.java b/src/main/java/ch/njol/skript/variables/MySQLStorage.java deleted file mode 100644 index ff5ac5d39ac..00000000000 --- a/src/main/java/ch/njol/skript/variables/MySQLStorage.java +++ /dev/null @@ -1,38 +0,0 @@ -package ch.njol.skript.variables; - -import ch.njol.skript.config.SectionNode; -import ch.njol.skript.log.SkriptLogger; -import lib.PatPeter.SQLibrary.Database; -import lib.PatPeter.SQLibrary.MySQL; - -public class MySQLStorage extends SQLStorage { - - MySQLStorage(String type) { - super(type, "CREATE TABLE IF NOT EXISTS %s (" + - "rowid BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY," + - "name VARCHAR(" + MAX_VARIABLE_NAME_LENGTH + ") NOT NULL UNIQUE," + - "type VARCHAR(" + MAX_CLASS_CODENAME_LENGTH + ")," + - "value BLOB(" + MAX_VALUE_SIZE + ")," + - "update_guid CHAR(36) NOT NULL" + - ") CHARACTER SET ucs2 COLLATE ucs2_bin"); - } - - @Override - public Database initialize(SectionNode config) { - String host = getValue(config, "host"); - Integer port = getValue(config, "port", Integer.class); - String user = getValue(config, "user"); - String password = getValue(config, "password"); - String database = getValue(config, "database"); - setTableName(config.get("table", "variables21")); - if (host == null || port == null || user == null || password == null || database == null) - return null; - return new MySQL(SkriptLogger.LOGGER, "[Skript]", host, port, database, user, password); - } - - @Override - protected boolean requiresFile() { - return false; - } - -} diff --git a/src/main/java/ch/njol/skript/variables/SQLStorage.java b/src/main/java/ch/njol/skript/variables/SQLStorage.java deleted file mode 100644 index e0ab25c3ee8..00000000000 --- a/src/main/java/ch/njol/skript/variables/SQLStorage.java +++ /dev/null @@ -1,588 +0,0 @@ -package ch.njol.skript.variables; - -import java.io.File; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.UUID; -import java.util.concurrent.Callable; - -import org.bukkit.Bukkit; -import org.bukkit.plugin.Plugin; -import org.jetbrains.annotations.Nullable; - -import ch.njol.skript.Skript; -import ch.njol.skript.classes.ClassInfo; -import ch.njol.skript.classes.Serializer; -import ch.njol.skript.config.SectionNode; -import ch.njol.skript.log.SkriptLogger; -import ch.njol.skript.registrations.Classes; -import ch.njol.skript.util.Task; -import ch.njol.skript.util.Timespan; -import ch.njol.util.SynchronizedReference; -import lib.PatPeter.SQLibrary.Database; -import lib.PatPeter.SQLibrary.DatabaseException; -import lib.PatPeter.SQLibrary.SQLibrary; - -/** - * TODO create a metadata table to store some properties (e.g. Skript version, Yggdrasil version) -- but what if some variables cannot be converted? move them to a different table? - * TODO create my own database connector or find a better one - * - * @author Peter Güttinger - */ -public abstract class SQLStorage extends VariablesStorage { - - public final static int MAX_VARIABLE_NAME_LENGTH = 380, // MySQL: 767 bytes max; cannot set max bytes, only max characters - MAX_CLASS_CODENAME_LENGTH = 50, // checked when registering a class - MAX_VALUE_SIZE = 10000; - - private final static String SELECT_ORDER = "name, type, value, rowid"; - - private final static String OLD_TABLE_NAME = "variables"; - - @Nullable - private String formattedCreateQuery; - private final String createTableQuery; - private String tableName; - - final SynchronizedReference db = new SynchronizedReference<>(null); - - private boolean monitor = false; - long monitor_interval; - - private final static String guid = UUID.randomUUID().toString(); - - /** - * The delay between transactions in milliseconds. - */ - private final static long TRANSACTION_DELAY = 500; - - /** - * Creates a SQLStorage with a create table query. - * - * @param type The database type i.e. CSV. - * @param createTableQuery The create table query to send to the SQL engine. - */ - public SQLStorage(String type, String createTableQuery) { - super(type); - this.createTableQuery = createTableQuery; - this.tableName = "variables21"; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - /** - * Initializes an SQL database with the user provided configuration section for loading the database. - * - * @param config The configuration from the config.sk that defines this database. - * @return A Database implementation from SQLibrary. - */ - @Nullable - public abstract Database initialize(SectionNode config); - - /** - * Retrieve the create query with the tableName in it - * @return the create query with the tableName in it (%s -> tableName) - */ - @Nullable - public String getFormattedCreateQuery() { - if (formattedCreateQuery == null) - formattedCreateQuery = String.format(createTableQuery, tableName); - return formattedCreateQuery; - } - - /** - * Doesn't lock the database for reading (it's not used anywhere else, and locking while loading will interfere with loaded variables being deleted by - * {@link Variables#variableLoaded(String, Object, VariablesStorage)}). - */ - @Override - protected boolean load_i(SectionNode n) { - synchronized (db) { - Plugin plugin = Bukkit.getPluginManager().getPlugin("SQLibrary"); - if (plugin == null || !(plugin instanceof SQLibrary)) { - Skript.error("You need the plugin SQLibrary in order to use a database with Skript. You can download the latest version from https://dev.bukkit.org/projects/sqlibrary/files/"); - return false; - } - - final Boolean monitor_changes = getValue(n, "monitor changes", Boolean.class); - final Timespan monitor_interval = getValue(n, "monitor interval", Timespan.class); - if (monitor_changes == null || monitor_interval == null) - return false; - monitor = monitor_changes; - this.monitor_interval = monitor_interval.getAs(Timespan.TimePeriod.MILLISECOND); - - final Database db; - try { - Database database = initialize(n); - if (database == null) - return false; - this.db.set(db = database); - } catch (final RuntimeException e) { - if (e instanceof DatabaseException) {// not in a catch clause to not produce a ClassNotFoundException when this class is loaded and SQLibrary is not present - Skript.error(e.getLocalizedMessage()); - return false; - } - throw e; - } - - SkriptLogger.setNode(null); - - if (!connect(true)) - return false; - - try { - final boolean hasOldTable = false; - final boolean hadNewTable = db.isTable(getTableName()); - - if (getFormattedCreateQuery() == null){ - Skript.error("Could not create the variables table in the database. The query to create the variables table '" + tableName + "' in the database '" + getUserConfigurationName() + "' is null."); - return false; - } - - try { - db.query(getFormattedCreateQuery()); - } catch (final SQLException e) { - Skript.error("Could not create the variables table '" + tableName + "' in the database '" + getUserConfigurationName() + "': " + e.getLocalizedMessage() + ". " - + "Please create the table yourself using the following query: " + String.format(createTableQuery, tableName).replace(",", ", ").replaceAll("\\s+", " ")); - return false; - } - - if (!prepareQueries()) { - return false; - } - - // old - // Table name support was added after the verison that used the legacy database format - - // new - final ResultSet r2 = db.query("SELECT " + SELECT_ORDER + " FROM " + getTableName()); - assert r2 != null; - try { - loadVariables(r2); - } finally { - r2.close(); - } - - // store old variables in new table and delete the old table - } catch (final SQLException e) { - sqlException(e); - return false; - } - - // periodically executes queries to keep the collection alive - Skript.newThread(new Runnable() { - @Override - public void run() { - while (!closed) { - synchronized (SQLStorage.this.db) { - try { - final Database db = SQLStorage.this.db.get(); - if (db != null) - db.query("SELECT * FROM " + getTableName() + " LIMIT 1"); - } catch (final SQLException e) {} - } - try { - Thread.sleep(1000 * 10); - } catch (final InterruptedException e) {} - } - } - }, "Skript database '" + getUserConfigurationName() + "' connection keep-alive thread").start(); - - return true; - } - } - - @Override - protected void allLoaded() { - Skript.debug("Database " + getUserConfigurationName() + " loaded. Queue size = " + changesQueue.size()); - - // start committing thread. Its first execution will also commit the first batch of changed variables. - Skript.newThread(new Runnable() { - @Override - public void run() { - long lastCommit; - while (!closed) { - synchronized (db) { - final Database db = SQLStorage.this.db.get(); - try { - if (db != null) - db.getConnection().commit(); - } catch (final SQLException e) { - sqlException(e); - } - lastCommit = System.currentTimeMillis(); - } - try { - Thread.sleep(Math.max(0, lastCommit + TRANSACTION_DELAY - System.currentTimeMillis())); - } catch (final InterruptedException e) {} - } - } - }, "Skript database '" + getUserConfigurationName() + "' transaction committing thread").start(); - - if (monitor) { - Skript.newThread(new Runnable() { - @Override - public void run() { - try { // variables were just downloaded, not need to check for modifications straight away - Thread.sleep(monitor_interval); - } catch (final InterruptedException e1) {} - - long lastWarning = Long.MIN_VALUE; - final int WARING_INTERVAL = 10; - - while (!closed) { - final long next = System.currentTimeMillis() + monitor_interval; - checkDatabase(); - final long now = System.currentTimeMillis(); - if (next < now && lastWarning + WARING_INTERVAL * 1000 < now) { - // TODO don't print this message when Skript loads (because scripts are loaded after variables and take some time) - Skript.warning("Cannot load variables from the database fast enough (loading took " + ((now - next + monitor_interval) / 1000.) + "s, monitor interval = " + (monitor_interval / 1000.) + "s). " + - "Please increase your monitor interval or reduce usage of variables. " + - "(this warning will be repeated at most once every " + WARING_INTERVAL + " seconds)"); - lastWarning = now; - } - while (System.currentTimeMillis() < next) { - try { - Thread.sleep(next - System.currentTimeMillis()); - } catch (final InterruptedException e) {} - } - } - } - }, "Skript database '" + getUserConfigurationName() + "' monitor thread").start(); - } - - } - - @Override - protected File getFile(String file) { - if (!file.endsWith(".db")) - file = file + ".db"; // required by SQLibrary - return new File(file); - } - - @Override - protected boolean connect() { - return connect(false); - } - - private final boolean connect(final boolean first) { - synchronized (db) { - // isConnected doesn't work in SQLite -// if (db.isConnected()) -// return; - final Database db = this.db.get(); - if (db == null || !db.open()) { - if (first) - Skript.error("Cannot connect to the database '" + getUserConfigurationName() + "'! Please make sure that all settings are correct");// + (type == Type.MYSQL ? " and that the database software is running" : "") + "."); - else - Skript.exception("Cannot reconnect to the database '" + getUserConfigurationName() + "'!"); - return false; - } - try { - db.getConnection().setAutoCommit(false); - } catch (final SQLException e) { - sqlException(e); - return false; - } - return true; - } - } - - /** - * (Re)creates prepared statements as they get closed as well when closing the connection - * - * @return - */ - private boolean prepareQueries() { - synchronized (db) { - final Database db = this.db.get(); - assert db != null; - try { - try { - if (writeQuery != null) - writeQuery.close(); - } catch (final SQLException e) {} - writeQuery = db.prepare("REPLACE INTO " + getTableName() + " (name, type, value, update_guid) VALUES (?, ?, ?, ?)"); - - try { - if (deleteQuery != null) - deleteQuery.close(); - } catch (final SQLException e) {} - deleteQuery = db.prepare("DELETE FROM " + getTableName() + " WHERE name = ?"); - - try { - if (monitorQuery != null) - monitorQuery.close(); - } catch (final SQLException e) {} - monitorQuery = db.prepare("SELECT " + SELECT_ORDER + " FROM " + getTableName() + " WHERE rowid > ? AND update_guid != ?"); - try { - if (monitorCleanUpQuery != null) - monitorCleanUpQuery.close(); - } catch (final SQLException e) {} - monitorCleanUpQuery = db.prepare("DELETE FROM " + getTableName() + " WHERE value IS NULL AND rowid < ?"); - } catch (final SQLException e) { - Skript.exception(e, "Could not prepare queries for the database '" + getUserConfigurationName() + "': " + e.getLocalizedMessage()); - return false; - } - } - return true; - } - - @Override - protected void disconnect() { - synchronized (db) { - final Database db = this.db.get(); -// if (!db.isConnected()) -// return; - if (db != null) - db.close(); - } - } - - /** - * Params: name, type, value, GUID - *

- * Writes a variable to the database - */ - @Nullable - private PreparedStatement writeQuery; - /** - * Params: name - *

- * Deletes a variable from the database - */ - @Nullable - private PreparedStatement deleteQuery; - /** - * Params: rowID, GUID - *

- * Selects changed rows. values in order: {@value #SELECT_ORDER} - */ - @Nullable - private PreparedStatement monitorQuery; - /** - * Params: rowID - *

- * Deletes null variables from the database older than the given value - */ - @Nullable - PreparedStatement monitorCleanUpQuery; - - @Override - protected boolean save(final String name, final @Nullable String type, final @Nullable byte[] value) { - synchronized (db) { - // REMIND get the actual maximum size from the database - if (name.length() > MAX_VARIABLE_NAME_LENGTH) - Skript.error("The name of the variable {" + name + "} is too long to be saved in a database (length: " + name.length() + ", maximum allowed: " + MAX_VARIABLE_NAME_LENGTH + ")! It will be truncated and won't bet available under the same name again when loaded."); - if (value != null && value.length > MAX_VALUE_SIZE) - Skript.error("The variable {" + name + "} cannot be saved in the database as its value's size (" + value.length + ") exceeds the maximum allowed size of " + MAX_VALUE_SIZE + "! An attempt to save the variable will be made nonetheless."); - try { - if (type == null) { - assert value == null; - final PreparedStatement deleteQuery = this.deleteQuery; - assert deleteQuery != null; - deleteQuery.setString(1, name); - deleteQuery.executeUpdate(); - } else { - int i = 1; - final PreparedStatement writeQuery = this.writeQuery; - assert writeQuery != null; - writeQuery.setString(i++, name); - writeQuery.setString(i++, type); - writeQuery.setBytes(i++, value); // SQLite desn't support setBlob - writeQuery.setString(i++, guid); - writeQuery.executeUpdate(); - } - } catch (final SQLException e) { - sqlException(e); - return false; - } - } - return true; - } - - @Override - public void close() { - synchronized (db) { - super.close(); - final Database db = this.db.get(); - if (db != null) { - try { - db.getConnection().commit(); - } catch (final SQLException e) { - sqlException(e); - } - db.close(); - this.db.set(null); - } - } - } - - long lastRowID = -1; - - protected void checkDatabase() { - try { - final long lastRowID; // local variable as this is used to clean the database below - ResultSet r = null; - try { - synchronized (db) { - if (closed || db.get() == null) - return; - lastRowID = this.lastRowID; - final PreparedStatement monitorQuery = this.monitorQuery; - assert monitorQuery != null; - monitorQuery.setLong(1, lastRowID); - monitorQuery.setString(2, guid); - monitorQuery.execute(); - r = monitorQuery.getResultSet(); - assert r != null; - } - if (!closed) - loadVariables(r); - } finally { - if (r != null) - r.close(); - } - - if (!closed) { // Skript may have been disabled in the meantime // TODO not fixed - new Task(Skript.getInstance(), (long) Math.ceil(2. * monitor_interval / 50) + 100, true) { // 2 times the interval + 5 seconds - @Override - public void run() { - try { - synchronized (db) { - if (closed || db.get() == null) - return; - final PreparedStatement monitorCleanUpQuery = SQLStorage.this.monitorCleanUpQuery; - assert monitorCleanUpQuery != null; - monitorCleanUpQuery.setLong(1, lastRowID); - monitorCleanUpQuery.executeUpdate(); - } - } catch (final SQLException e) { - sqlException(e); - } - } - }; - } - } catch (final SQLException e) { - sqlException(e); - } - } - -// private final static class VariableInfo { -// final String name; -// final byte[] value; -// final ClassInfo ci; -// -// public VariableInfo(final String name, final byte[] value, final ClassInfo ci) { -// this.name = name; -// this.value = value; -// this.ci = ci; -// } -// } - -// final static LinkedList syncDeserializing = new LinkedList(); - - /** - * Doesn't lock the database - {@link #save(String, String, byte[])} does that // what? - */ - private void loadVariables(final ResultSet r) throws SQLException { -// assert !Thread.holdsLock(db); -// synchronized (syncDeserializing) { - - final SQLException e = Task.callSync(new Callable() { - @Override - @Nullable - public SQLException call() throws Exception { - try { - while (r.next()) { - int i = 1; - final String name = r.getString(i++); - if (name == null) { - Skript.error("Variable with NULL name found in the database '" + getUserConfigurationName() + "', ignoring it"); - continue; - } - final String type = r.getString(i++); - final byte[] value = r.getBytes(i++); // Blob not supported by SQLite - lastRowID = r.getLong(i++); - if (value == null) { - Variables.variableLoaded(name, null, SQLStorage.this); - } else { - final ClassInfo c = Classes.getClassInfoNoError(type); - @SuppressWarnings("unused") - Serializer s; - if (c == null || (s = c.getSerializer()) == null) { - Skript.error("Cannot load the variable {" + name + "} from the database '" + getUserConfigurationName() + "', because the type '" + type + "' cannot be recognised or cannot be stored in variables"); - continue; - } -// if (s.mustSyncDeserialization()) { -// syncDeserializing.add(new VariableInfo(name, value, c)); -// } else { - final Object d = Classes.deserialize(c, value); - if (d == null) { - Skript.error("Cannot load the variable {" + name + "} from the database '" + getUserConfigurationName() + "', because it cannot be loaded as " + c.getName().withIndefiniteArticle()); - continue; - } - Variables.variableLoaded(name, d, SQLStorage.this); -// } - } - } - } catch (final SQLException e) { - return e; - } - return null; - } - }); - if (e != null) - throw e; - -// if (!syncDeserializing.isEmpty()) { -// Task.callSync(new Callable() { -// @Override -// @Nullable -// public Void call() throws Exception { -// synchronized (syncDeserializing) { -// for (final VariableInfo o : syncDeserializing) { -// final Object d = Classes.deserialize(o.ci, o.value); -// if (d == null) { -// Skript.error("Cannot load the variable {" + o.name + "} from the database " + getUserConfigurationName() + ", because it cannot be loaded as a " + o.ci.getName()); -// continue; -// } -// Variables.variableLoaded(o.name, d, DatabaseStorage.this); -// } -// syncDeserializing.clear(); -// return null; -// } -// } -// }); -// } -// } - } - -// private final static class OldVariableInfo { -// final String name; -// final String value; -// final ClassInfo ci; -// -// public OldVariableInfo(final String name, final String value, final ClassInfo ci) { -// this.name = name; -// this.value = value; -// this.ci = ci; -// } -// } - -// final static LinkedList oldSyncDeserializing = new LinkedList(); - - void sqlException(final SQLException e) { - Skript.error("database error: " + e.getLocalizedMessage()); - if (Skript.testing()) - e.printStackTrace(); - prepareQueries(); // a query has to be recreated after an error - } - -} diff --git a/src/main/java/ch/njol/skript/variables/SQLiteStorage.java b/src/main/java/ch/njol/skript/variables/SQLiteStorage.java deleted file mode 100644 index 57032218f95..00000000000 --- a/src/main/java/ch/njol/skript/variables/SQLiteStorage.java +++ /dev/null @@ -1,37 +0,0 @@ -package ch.njol.skript.variables; - -import java.io.File; - -import ch.njol.skript.config.SectionNode; -import ch.njol.skript.log.SkriptLogger; -import lib.PatPeter.SQLibrary.Database; -import lib.PatPeter.SQLibrary.SQLite; - -public class SQLiteStorage extends SQLStorage { - - SQLiteStorage(String type) { - super(type, "CREATE TABLE IF NOT EXISTS %s (" + - "name VARCHAR(" + MAX_VARIABLE_NAME_LENGTH + ") NOT NULL PRIMARY KEY," + - "type VARCHAR(" + MAX_CLASS_CODENAME_LENGTH + ")," + - "value BLOB(" + MAX_VALUE_SIZE + ")," + - "update_guid CHAR(36) NOT NULL" + - ")"); - } - - @Override - public Database initialize(SectionNode config) { - File f = file; - if (f == null) - return null; - setTableName(config.get("table", "variables21")); - String name = f.getName(); - assert name.endsWith(".db"); - return new SQLite(SkriptLogger.LOGGER, "[Skript]", f.getParent(), name.substring(0, name.length() - ".db".length())); - } - - @Override - protected boolean requiresFile() { - return true; - } - -} diff --git a/src/main/java/ch/njol/skript/variables/SerializedVariable.java b/src/main/java/ch/njol/skript/variables/SerializedVariable.java index 258a251a9b5..3d65d5a76d4 100644 --- a/src/main/java/ch/njol/skript/variables/SerializedVariable.java +++ b/src/main/java/ch/njol/skript/variables/SerializedVariable.java @@ -5,58 +5,24 @@ /** * An instance of a serialized variable, contains the variable name * and the serialized value. + * + * @param name The name of the variable. + * @param value The serialized value of the variable. + * A value of {@code null} indicates the variable will be deleted. */ -public class SerializedVariable { +public record SerializedVariable(String name, @Nullable Value value) { - /** - * The name of the variable. - */ - public final String name; - - /** - * The serialized value of the variable. - *

- * A value of {@code null} indicates the variable will be deleted. - */ - @Nullable - public final Value value; - - /** - * Creates a new serialized variable with the given name and value. - * - * @param name the given name. - * @param value the given value, or {@code null} to indicate a deletion. - */ - public SerializedVariable(String name, @Nullable Value value) { - this.name = name; - this.value = value; + public SerializedVariable(String name, String type, byte[] data) { + this(name, new Value(type, data)); } /** * A serialized value of a variable. + * + * @param type The type of this value. + * @param data The serialized value data. */ - public static final class Value { - - /** - * The type of this value. - */ - public final String type; - - /** - * The serialized value data. - */ - public final byte[] data; - - /** - * Creates a new serialized value. - * @param type the value type. - * @param data the serialized value data. - */ - public Value(String type, byte[] data) { - this.type = type; - this.data = data; - } - + public record Value(String type, byte[] data) { } } diff --git a/src/main/java/ch/njol/skript/variables/UnloadedStorage.java b/src/main/java/ch/njol/skript/variables/UnloadedStorage.java new file mode 100644 index 00000000000..bfa1ed07f33 --- /dev/null +++ b/src/main/java/ch/njol/skript/variables/UnloadedStorage.java @@ -0,0 +1,54 @@ +package ch.njol.skript.variables; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jspecify.annotations.NonNull; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.util.Arrays; +import java.util.function.BiFunction; + +/** + * Represents an unloaded storage type for variables. + * This class stores all the data from register time to be used if this database is selected. + * + * @param source The SkriptAddon that is registering this storage type. + * @param storage The class of the actual VariableStorage to initalize with. + * @param names The possible user input names from the config.sk to match this storage. + */ +@ApiStatus.Internal +record UnloadedStorage(SkriptAddon source, Class storage, + BiFunction constructor, String... names) { + + /** + * Creates new variable storage instance of this type. + * + * @param addon addon that created this storage instance + * @param name the database type i.e. CSV. + * @return variable storage of this type + */ + public T create(SkriptAddon addon, String name) { + return constructor.apply(addon, name); + } + + @Override + @Contract(value = " -> new", pure = true) + public String @NonNull [] names() { + return Arrays.copyOf(names, names.length); + } + + /** + * Checks if a user input matches this storage input names. + * + * @param input The name to check against. + * @return true if this storage matches the user input, otherwise false. + */ + public boolean matches(String input) { + for (String name : names) { + if (name.equalsIgnoreCase(input)) + return true; + } + return false; + } + +} diff --git a/src/main/java/ch/njol/skript/variables/VariableStorage.java b/src/main/java/ch/njol/skript/variables/VariableStorage.java new file mode 100644 index 00000000000..18b9aa7b3d4 --- /dev/null +++ b/src/main/java/ch/njol/skript/variables/VariableStorage.java @@ -0,0 +1,360 @@ +package ch.njol.skript.variables; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import com.google.errorprone.annotations.ThreadSafe; +import org.jetbrains.annotations.Blocking; +import org.jetbrains.annotations.Nullable; +import ch.njol.skript.Skript; +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.lang.ParseContext; +import ch.njol.skript.log.ParseLogHandler; +import ch.njol.skript.log.SkriptLogger; +import ch.njol.skript.registrations.Classes; +import org.jetbrains.annotations.VisibleForTesting; +import org.skriptlang.skript.addon.SkriptAddon; + +/** + * A variable storage is holds the means and methods of storing variables. + * This is usually some sort of database, and could be as simply as a text file. + *

+ * Variable storage itself is responsible for variable management, read and write requests, + * and loading and unloading the variables to memory. + *

+ * For storing variables on heap, see {@link VariablesMap} which provides thread safe + * implementation for on heap variable storage. + */ +@ThreadSafe +public abstract class VariableStorage implements Closeable { + + /** + * Source of the variable storage. + */ + private final SkriptAddon source; + + /** + * The name of the database + */ + private String databaseName; + + /** + * The type of the database, i.e. CSV. + */ + private final String databaseType; + + /** + * The file associated with this variable storage. + * Can be {@code null} if no file is required. + */ + protected @Nullable File file; + + /** + * The pattern of the variable name this storage accepts. + * {@code null} for '{@code .*}' or '{@code .*}'. + */ + private @Nullable Pattern variableNamePattern; + + protected VariableStorage(SkriptAddon source, String type) { + assert type != null; + this.source = source; + this.databaseType = type; + } + + /** + * Get the config name of a database + *

+ * Note: Returns the user set name for the database, ex: + * 

{@code
+	 * default: <- Config Name
+	 *    type: CSV
+	 * }
+ * @return name of database + */ + protected final String getUserConfigurationName() { + return databaseName; + } + + /** + * Get the config type of a database + * + * @return type of database + */ + protected final String getDatabaseType() { + return databaseType; + } + + /** + * @return The SkriptAddon instance that registered this VariableStorage. + */ + public final SkriptAddon getRegisterSource() { + return source; + } + + /** + * Gets the string value at the given key of the given section node. + * + * @param sectionNode the section node. + * @param key the key. + * @return the value, or {@code null} if the value was invalid, + * or not found. + */ + protected final @Nullable String getValue(SectionNode sectionNode, String key) { + return getValue(sectionNode, key, String.class); + } + + /** + * Gets the value at the given key of the given section node, + * parsed with the given type. + * + * @param sectionNode the section node. + * @param key the key. + * @param type the type. + * @return the parsed value, or {@code null} if the value was invalid, + * or not found. + * @param the type. + */ + protected final @Nullable T getValue(SectionNode sectionNode, String key, Class type) { + return getValue(sectionNode, key, type, true); + } + + /** + * Gets the value at the given key of the given section node, + * parsed with the given type. Prints no errors, but can return null. + * + * @param sectionNode the section node. + * @param key the key. + * @param type the type. + * @return the parsed value, or {@code null} if the value was invalid, + * or not found. + * @param the type. + */ + protected final @Nullable T getOptional(SectionNode sectionNode, String key, Class type) { + return getValue(sectionNode, key, type, false); + } + + /** + * Gets the value at the given key of the given section node, + * parsed with the given type. + * + * @param sectionNode the section node. + * @param key the key. + * @param type the type. + * @param error if Skript should print errors and stop loading. + * @return the parsed value, or {@code null} if the value was invalid, + * or not found. + * @param the type. + */ + private @Nullable T getValue(SectionNode sectionNode, String key, Class type, boolean error) { + String rawValue = sectionNode.getValue(key); + if (rawValue == null) { + if (error) + Skript.error("The config is missing the entry for '" + key + "' in the database '" + databaseName + "'"); + return null; + } + + try (ParseLogHandler log = SkriptLogger.startParseLogHandler()) { + T parsedValue = Classes.parse(rawValue, type, ParseContext.CONFIG); + if (parsedValue == null && error) + // Parsing failed + log.printError("The entry for '" + key + "' in the database '" + databaseName + "' must be " + + Classes.getSuperClassInfo(type).getName().withIndefiniteArticle()); + else + log.printLog(); + + return parsedValue; + } + } + + private static final Set registeredFiles = ConcurrentHashMap.newKeySet(); + + /** + * Loads the configuration for this variable storage + * from the given section node. Loads internal required values first in loadConfig. + * {@link #load(SectionNode)} is for extending classes. + *

+ * This operation may be blocking if the variable storage deserializes some stored variables + * as some deserializers must be synced on the main thread. + * + * @param sectionNode the section node. + * @return whether the loading succeeded. + */ + @Blocking + @VisibleForTesting + public final boolean loadConfig(SectionNode sectionNode) { + databaseName = sectionNode.getKey(); + String pattern = getValue(sectionNode, "pattern"); + if (pattern == null) + return false; + + try { + // Set variable name pattern, see field javadoc for explanation of null value + variableNamePattern = pattern.equals(".*") || pattern.equals(".+") ? null : Pattern.compile(pattern); + } catch (PatternSyntaxException e) { + Skript.error("Invalid pattern '" + pattern + "': " + e.getLocalizedMessage()); + return false; + } + + if (requiresFile()) { + // Initialize file + String fileName = getValue(sectionNode, "file"); + if (fileName == null) + return false; + + file = getFile(fileName).getAbsoluteFile(); + + if (file.exists() && !file.isFile()) { + Skript.error("The database file '" + file.getName() + "' must be an actual file, not a directory."); + return false; + } + + // Create the file if it does not exist yet + try { + if (!file.exists() && !file.createNewFile()) { + Skript.error("Cannot create the database file '" + file.getName() + "'"); + return false; + } + } catch (IOException e) { + Skript.error("Cannot create the database file '" + file.getName() + "': " + e.getLocalizedMessage()); + return false; + } + + // Check for read & write permissions to the file + if (!file.canWrite()) { + Skript.error("Cannot write to the database file '" + file.getName() + "'!"); + return false; + } + if (!file.canRead()) { + Skript.error("Cannot read from the database file '" + file.getName() + "'!"); + return false; + } + + if (registeredFiles.contains(file)) { + Skript.error("Database `" + databaseName + "` failed to load. The file `" + fileName + "` is already registered to another database."); + return false; + } + registeredFiles.add(file); + } + + // Load the entries custom to the variable storage + return loadAbstract(sectionNode); + } + + /** + * Used for abstract extending classes intercepting the + * configuration before sending to the final implementation class. + *

+ * Override to use this method in AnotherAbstractClass; + * VariablesStorage -> AnotherAbstractClass -> FinalImplementation + *

+ * This operation may be blocking if the variable storage deserializes some stored variables + * as some deserializers must be synced on the main thread. + * + * @param sectionNode the section node. + * @return whether the loading succeeded. + */ + @Blocking + protected boolean loadAbstract(SectionNode sectionNode) { + return load(sectionNode); + } + + /** + * Loads variables stored here. + *

+ * This operation may be blocking if the variable storage deserializes some stored variables + * as some deserializers must be synced on the main thread. + * + * @param sectionNode the section node. + * @return Whether the database could be loaded successfully, + * i.e. whether the config is correct and all variables could be loaded. + */ + @Blocking + protected abstract boolean load(SectionNode sectionNode); + + /** + * Checks if this storage requires a file for storing its data. + * + * @return if this storage needs a file. + */ + protected abstract boolean requiresFile(); + + /** + * Gets the file needed for this variable storage from the given file name. + *

+ * Will only be called if {@link #requiresFile()} is {@code true}. + * + * @param fileName the given file name. + * @return the {@link File} object. + */ + protected abstract File getFile(String fileName); + + /** + * Reads a variable with given name from this storage. + *

+ * The format of returned value follows {@link VariablesMap#getVariable(String)}. + *

+ * This method must be thread safe. + * + * @param name name of the variable + * @return value of the variable + */ + public abstract @Nullable Object getVariable(String name); + + /** + * Sets the given variable to the given value. + *

+ * This method accepts list variables, + * but these may only be set to {@code null}. + *

+ * This method must be thread safe. + * + * @param name the variable name. + * @param value the variable value, {@code null} to delete the variable. + */ + public abstract void setVariable(String name, @Nullable Object value); + + /** + * Returns the number of currently loaded variables. + *

+ * This number may not be fully accurate. + * + * @return number of loaded variables + */ + public abstract long loadedVariables(); + + // TODO backups + + /** + * Checks if this variable storage accepts the given variable name. + * + * @param var the variable name. + * @return if this storage accepts the variable name. + * @see #variableNamePattern + */ + public boolean accept(@Nullable String var) { + if (var == null) + return false; + return variableNamePattern == null || variableNamePattern.matcher(var).matches(); + } + + /** + * Returns the name pattern accepted by this variable storage + * + * @return the name pattern, or null if accepting all + */ + public @Nullable Pattern getNamePattern() { + return variableNamePattern; + } + + /** + * Called when Skript gets disabled. + */ + @Override + public abstract void close(); + +} diff --git a/src/main/java/ch/njol/skript/variables/Variables.java b/src/main/java/ch/njol/skript/variables/Variables.java index 9e0a36bc57f..b9d688ea888 100644 --- a/src/main/java/ch/njol/skript/variables/Variables.java +++ b/src/main/java/ch/njol/skript/variables/Variables.java @@ -8,58 +8,52 @@ import ch.njol.skript.config.Config; import ch.njol.skript.config.Node; import ch.njol.skript.config.SectionNode; +import ch.njol.skript.lang.KeyedValue; import ch.njol.skript.lang.Variable; import ch.njol.skript.log.SkriptLogger; import ch.njol.skript.registrations.Classes; -import ch.njol.skript.variables.SerializedVariable.Value; -import ch.njol.util.Kleenean; -import ch.njol.util.NonNullPair; -import ch.njol.util.Pair; -import ch.njol.util.StringUtils; -import ch.njol.util.SynchronizedReference; +import ch.njol.util.*; import ch.njol.util.coll.iterator.EmptyIterator; +import ch.njol.yggdrasil.ClassResolver; import ch.njol.yggdrasil.Yggdrasil; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import org.bukkit.Bukkit; +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.errorprone.annotations.ThreadSafe; import org.bukkit.configuration.serialization.ConfigurationSerializable; import org.bukkit.configuration.serialization.ConfigurationSerialization; import org.bukkit.event.Event; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.UnmodifiableView; -import org.skriptlang.skript.lang.converter.Converters; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Queue; -import java.util.TreeMap; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.jetbrains.annotations.VisibleForTesting; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; import java.util.regex.Pattern; +import org.skriptlang.skript.variables.storage.H2Storage; +import org.skriptlang.skript.variables.storage.InMemoryVariableStorage; +import org.skriptlang.skript.variables.storage.MySQLStorage; +import org.skriptlang.skript.variables.storage.SQLiteStorage; + /** - * Handles all things related to variables. + * Factory class that handles things related to variables. + *

+ * This includes the registration and management of variable + * storage types and variable access. + *

+ * All methods in this class are thread safe but some may be blocking, see documentation. * * @see #setVariable(String, Object, Event, boolean) * @see #getVariable(String, Event, boolean) */ -public class Variables { +@ThreadSafe +public final class Variables { /** * The version of {@link Yggdrasil} this class is using. @@ -76,84 +70,85 @@ public class Variables { */ public static boolean caseInsensitiveVariables = true; + // registered storages types + private static final List> UNLOADED_STORAGES = Collections.synchronizedList(new ArrayList<>()); + // actually loaded storages types by the user + @VisibleForTesting + static final List STORAGES = new CopyOnWriteArrayList<>(); + /** - * The {@link ch.njol.yggdrasil.ClassResolver#getID(Class) ID} prefix - * for {@link ConfigurationSerializable} classes. + * Whether the storages specified by user has been loaded. + *

+ * If true, it is safe to access variables. */ - private static final String CONFIGURATION_SERIALIZABLE_PREFIX = "ConfigurationSerializable_"; - - private final static Multimap, String> TYPES = HashMultimap.create(); - - // Register some things with Yggdrasil - static { - registerStorage(FlatFileStorage.class, "csv", "file", "flatfile"); - registerStorage(SQLiteStorage.class, "sqlite"); - registerStorage(MySQLStorage.class, "mysql"); - yggdrasil.registerSingleClass(Kleenean.class, "Kleenean"); - // Register ConfigurationSerializable, Bukkit's serialization system - yggdrasil.registerClassResolver(new ConfigurationSerializer() { - { - //noinspection unchecked - info = (ClassInfo) (ClassInfo) Classes.getExactClassInfo(Object.class); - // Info field is mostly unused in superclass, due to methods overridden below, - // so this illegal cast is fine - } - - @Override - @Nullable - public String getID(@NotNull Class c) { - if (ConfigurationSerializable.class.isAssignableFrom(c) - && Classes.getSuperClassInfo(c) == Classes.getExactClassInfo(Object.class)) - return CONFIGURATION_SERIALIZABLE_PREFIX + - ConfigurationSerialization.getAlias(c.asSubclass(ConfigurationSerializable.class)); - - return null; - } - - @Override - @Nullable - public Class getClass(@NotNull String id) { - if (id.startsWith(CONFIGURATION_SERIALIZABLE_PREFIX)) - return ConfigurationSerialization.getClassByAlias( - id.substring(CONFIGURATION_SERIALIZABLE_PREFIX.length())); - - return null; - } - }); - } + private static final AtomicBoolean loaded = new AtomicBoolean(false); /** - * The variable storages configured. + * Variable storage that is used as a backup. + *

+ * Such variable storage must not have any variable pattern. */ - static final List STORAGES = new ArrayList<>(); + private static VariableStorage defaultStorage; /** - * @return a copy of the list of variable storage handlers + * Variable storage that is used for ephemeral variables. */ - public static @UnmodifiableView List getStores() { - return Collections.unmodifiableList(STORAGES); + private static VariableStorage ephemeralStorage; + + static { + registerSkriptStorageTypes(Skript.instance()); + yggdrasil.registerSingleClass(Kleenean.class, "Kleenean"); + yggdrasil.registerClassResolver(new BukkitConfigurationSerializer()); } /** * Register a VariableStorage class for Skript to create if the user config value matches. * - * @param A class to extend VariableStorage. + * @param source source of the storage type * @param storage The class of the VariableStorage implementation. + * @param constructor provider of the storage instance * @param names The names used in the config of Skript to select this VariableStorage. - * @return if the operation was successful, or if it's already registered. - */ - public static boolean registerStorage(Class storage, String... names) { - if (TYPES.containsKey(storage)) - return false; - for (String name : names) { - if (TYPES.containsValue(name.toLowerCase(Locale.ENGLISH))) + * @return whether the storage type registration was successful + * @param A class to extend VariableStorage. + * @throws SkriptAPIException if the operation was not successful because the storage class is already registered. + */ + public static boolean registerStorage(SkriptAddon source, Class storage, + BiFunction constructor, String... names) { + Skript.checkAcceptRegistrations(); + for (UnloadedStorage registered : UNLOADED_STORAGES) { + if (registered.storage().isAssignableFrom(storage)) + throw new SkriptAPIException("Storage class '" + storage.getName() + "' cannot be registered because '" + + registered.storage().getName() + "' is a superclass or equal class"); + if (Arrays.stream(names).anyMatch(registered::matches)) return false; } - for (String name : names) - TYPES.put(storage, name.toLowerCase(Locale.ENGLISH)); + UNLOADED_STORAGES.add(new UnloadedStorage<>(source, storage, constructor, names)); return true; } + /** + * Registers the default storage types provided by Skript. + * + * @param source source for the registration + */ + private static void registerSkriptStorageTypes(SkriptAddon source) { + registerStorage(source, FlatFileStorage.class, FlatFileStorage::new, "csv", "file", "flatfile"); + if (Skript.classExists("com.zaxxer.hikari.HikariConfig")) { + registerStorage(source, SQLiteStorage.class, SQLiteStorage::new, "sqlite"); + registerStorage(source, MySQLStorage.class, MySQLStorage::new, "mysql"); + registerStorage(source, H2Storage.class, H2Storage::new, "h2"); + } else { + Skript.warning("SpigotLibraryLoader failed to load HikariCP. No JDBC databases were enabled."); + } + } + + /** + * @return a copy of the list of variable storage handlers + */ + public static @UnmodifiableView List getLoadedStorages() { + return Collections.unmodifiableList(STORAGES); + } + /** * Load the variables configuration and all variables. *

@@ -162,9 +157,9 @@ public static boolean registerStorage(Class stor * @return whether the loading was successful. */ public static boolean load() { - assert variables.treeMap.isEmpty(); - assert variables.hashMap.isEmpty(); assert STORAGES.isEmpty(); + if (loaded.compareAndExchange(false, true)) + throw new SkriptAPIException("Variables already loaded"); Config config = SkriptConfig.getConfig(); if (config == null) @@ -176,129 +171,99 @@ public static boolean load() { return false; } + //noinspection removal Skript.closeOnDisable(Variables::close); - // reports once per second how many variables were loaded. Useful to make clear that Skript is still doing something if it's loading many variables - Thread loadingLoggerThread = new Thread(() -> { - while (true) { - try { - Thread.sleep(Skript.logNormal() ? 1000 : 5000); // low verbosity won't disable these messages, but makes them more rare - } catch (InterruptedException ignored) {} - - synchronized (TEMP_VARIABLES) { - Map> tvs = TEMP_VARIABLES.get(); - if (tvs != null) - Skript.info("Loaded " + tvs.size() + " variables so far..."); - else - break; // variables loaded, exit thread - } - } - }); - loadingLoggerThread.start(); + boolean success = true; - try { - boolean successful = true; - - for (Node node : (SectionNode) databases) { - if (node instanceof SectionNode) { - SectionNode sectionNode = (SectionNode) node; - - String type = sectionNode.getValue("type"); - if (type == null) { - Skript.error("Missing entry 'type' in database definition"); - successful = false; - continue; - } - - String name = sectionNode.getKey(); - assert name != null; - - // Initiate the right VariablesStorage class - VariablesStorage variablesStorage; - Optional optional = TYPES.entries().stream() - .filter(entry -> entry.getValue().equalsIgnoreCase(type)) - .map(Entry::getKey) - .findFirst(); - if (!optional.isPresent()) { - if (!type.equalsIgnoreCase("disabled") && !type.equalsIgnoreCase("none")) { - Skript.error("Invalid database type '" + type + "'"); - successful = false; - } - continue; - } - - try { - @SuppressWarnings("unchecked") - Class storageClass = (Class) optional.get(); - Constructor constructor = storageClass.getDeclaredConstructor(String.class); - constructor.setAccessible(true); - variablesStorage = (VariablesStorage) constructor.newInstance(type); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - Skript.error("Failed to initialize database `" + name + "`"); - successful = false; - continue; - } - - // Get the amount of variables currently loaded - int totalVariablesLoaded; - synchronized (TEMP_VARIABLES) { - Map> tvs = TEMP_VARIABLES.get(); - assert tvs != null; - totalVariablesLoaded = tvs.size(); - } - - long start = System.currentTimeMillis(); - if (Skript.logVeryHigh()) - Skript.info("Loading database '" + node.getKey() + "'..."); - - // Load the variables - if (variablesStorage.load(sectionNode)) - STORAGES.add(variablesStorage); - else - successful = false; - - // Get the amount of variables loaded by this variables storage object - int newVariablesLoaded; - synchronized (TEMP_VARIABLES) { - Map> tvs = TEMP_VARIABLES.get(); - assert tvs != null; - newVariablesLoaded = tvs.size() - totalVariablesLoaded; - } - - if (Skript.logVeryHigh()) { - Skript.info("Loaded " + newVariablesLoaded + " variables from the database " + - "'" + sectionNode.getKey() + "' in " + - ((System.currentTimeMillis() - start) / 100) / 10.0 + " seconds"); - } - } else { - Skript.error("Invalid line in databases: databases must be defined as sections"); - successful = false; - } + for (Node node : databases) { + if (!(node instanceof SectionNode sectionNode)) { + Skript.error("Invalid line in databases: databases must be defined as sections"); + success = false; + continue; } - if (!successful) - return false; + // Databases must be loaded sequentially on this thread. + // Skript calls this method on the main thread, loading the databases in parallel and wait + // would cause a dead lock because deserialization of variables from databases must be synced + // on the main thread for some types. + if (!loadDatabase(sectionNode)) { + Skript.error("Failed to load database from node: " + sectionNode); + success = false; + } + } + + // TODO possibly migrate variables to according databases if the variable storage does + // not accept the name of the variable. + // Currently no migration happens and no variables are lost on shutdown but that + // will cause "rediscovery" of old variables when database patterns get changed. + // There should also exist a migration system to move from flat file storage to + // modern alternatives for users. + try { if (STORAGES.isEmpty()) { - Skript.error("No databases to store variables are defined. Please enable at least the default database, even if you don't use variables at all."); + Skript.error("No databases to store variables are defined. Please enable at least the default " + + "database, even if you don't use variables at all."); return false; } + List missingPattern = STORAGES.stream() + .filter(storage -> storage.getNamePattern() == null) + .toList(); + if (missingPattern.size() > 1) + Skript.warning("You have multiple databases with pattern accepting all variable names, " + + "Skript will attempt to save variables only to one of them."); + if (missingPattern.isEmpty()) { + Skript.warning("You have no database matching all variable names. Some of your variables " + + "will not save."); + } else { + defaultStorage = missingPattern.getFirst(); + } + return success; } finally { + // there must always be at least a default storage to store global variables somewhere + if (defaultStorage == null) + defaultStorage = new InMemoryVariableStorage(Skript.instance(), "in-memory"); + ephemeralStorage = new InMemoryVariableStorage(Skript.instance(), "in-memory ephemeral"); SkriptLogger.setNode(null); + } + } - // make sure to put the loaded variables into the variables map - int notStoredVariablesCount = onStoragesLoaded(); - if (notStoredVariablesCount != 0) { - Skript.warning(notStoredVariablesCount + " variables were possibly discarded due to not belonging to any database " + - "(SQL databases keep such variables and will continue to generate this warning, " + - "while CSV discards them)."); - } + /** + * Loads a database from a section node. + * + * @param node section node in the database config + * @return result + */ + private static boolean loadDatabase(SectionNode node) { + String type = node.getValue("type"); + if (type == null) { + Skript.error("Missing entry 'type' in database definition"); + return false; + } - // Interrupt the loading logger thread to make it exit earlier - loadingLoggerThread.interrupt(); + String name = node.getKey(); + assert name != null; - saveThread.start(); + Optional> optional = UNLOADED_STORAGES.stream() + .filter(registered -> registered.matches(type)) + .findFirst(); + + if (optional.isEmpty()) { + if (type.equalsIgnoreCase("disabled") || type.equalsIgnoreCase("none")) + return true; + Skript.error("Invalid database type '" + type + "'"); + return false; } - return true; + + UnloadedStorage unloadedStorage = optional.get(); + VariableStorage variablesStorage = unloadedStorage.create(unloadedStorage.source(), type); + + if (Skript.logVeryHigh()) + Skript.info("Loading database '" + name + "'..."); + + boolean result = variablesStorage.loadConfig(node); + if (result) + STORAGES.add(variablesStorage); + return result; } /** @@ -318,51 +283,18 @@ public static String[] splitVariableName(String name) { } /** - * A lock for reading and writing variables. - */ - static final ReadWriteLock variablesLock = new ReentrantReadWriteLock(true); - - /** - * The {@link VariablesMap} storing global variables, - * must be locked with {@link #variablesLock}. - */ - static final VariablesMap variables = new VariablesMap(); - - /** - * A map storing all local variables, + * A cache storing all local variables, * indexed by their {@link Event}. - */ - private static final Map localVariables = new ConcurrentHashMap<>(); - - /** - * Gets the {@link TreeMap} of all global variables. - *

- * Remember to lock with {@link #getReadLock()} and to not make any changes! - */ - static TreeMap getVariables() { - return variables.treeMap; - } - - /** - * Gets the {@link Map} of all global variables. *

- * This map cannot be modified. - * Remember to lock with {@link #getReadLock()}! - */ - static Map getVariablesHashMap() { - return Collections.unmodifiableMap(variables.hashMap); - } - - /** - * Gets the lock for reading variables. - * - * @return the lock. - * - * @see #variablesLock + * We use weak key cache because: + *

    + *
  • variables are not kept in memory for expired events
    • + *
  • variables map are looked up from event instance identity
    • + *
*/ - static Lock getReadLock() { - return variablesLock.readLock(); - } + private static final LoadingCache localVariables = CacheBuilder.newBuilder() + .weakKeys() + .build(CacheLoader.from(() -> new VariablesMap())); /** * Removes local variables associated with given event and returns them, @@ -372,9 +304,21 @@ static Lock getReadLock() { * @return the local variables from the event, * or {@code null} if the event had no local variables. */ - @Nullable public static VariablesMap removeLocals(Event event) { - return localVariables.remove(event); + return localVariables.asMap().remove(event); + } + + /** + * Returns the variable map for given event. + *

+ * This never returns null and provides new variables map if there is none for + * the given event. + * + * @param event event + * @return variables map of given event + */ + public static VariablesMap getLocalVariables(Event event) { + return localVariables.getUnchecked(event); } /** @@ -388,9 +332,9 @@ public static VariablesMap removeLocals(Event event) { * @param event the event. * @param map the new local variables. */ - public static void setLocalVariables(Event event, @Nullable Object map) { + public static void setLocalVariables(Event event, @Nullable VariablesMap map) { if (map != null) { - localVariables.put(event, (VariablesMap) map); + localVariables.put(event, map); } else { removeLocals(event); } @@ -403,22 +347,19 @@ public static void setLocalVariables(Event event, @Nullable Object map) { * @param event the event to copy local variables from. * @return the copy. */ - public static @Nullable Object copyLocalVariables(Event event) { - VariablesMap from = localVariables.get(event); - if (from == null) - return null; - - return from.copy(); + public static VariablesMap copyLocalVariables(Event event) { + return getLocalVariables(event).copy(); } /** * Copies local variables from provider to user, runs action, then copies variables back to provider. * Removes local variables from user after action is finished. + * * @param provider The originator of the local variables. * @param user The event to copy the variables to and back from. * @param action The code to run while the variables are copied. */ - public static void withLocalVariables(Event provider, Event user, @NotNull Runnable action) { + public static void withLocalVariables(Event provider, Event user, Runnable action) { Variables.setLocalVariables(user, Variables.copyLocalVariables(provider)); action.run(); Variables.setLocalVariables(provider, Variables.copyLocalVariables(user)); @@ -426,87 +367,99 @@ public static void withLocalVariables(Event provider, Event user, @NotNull Runna } /** - * Returns the internal value of the requested variable. + * Finds appropriate variable storage for given variable name. + * + * @param name name of the variable + * @return its preffered variable storage + * @see VariableStorage#accept(String) + */ + private static VariableStorage findStorage(String name) { + assert defaultStorage != null; + assert ephemeralStorage != null; + if (name.startsWith(Variable.EPHEMERAL_VARIABLE_TOKEN)) + return ephemeralStorage; + var all = STORAGES.stream() + .filter(storage -> storage != defaultStorage) // default storage is fallback + .filter(storage -> storage.accept(name)) + .toList(); + if (all.size() == 1) + return all.getFirst(); + if (!all.isEmpty()) { + VariableStorage first = all.getFirst(); + Skript.warning("Found multiple databases for variable '" + name + "'. Resolve the database pattern " + + "conflicts in the config. Saving to '" + first.getUserConfigurationName() + "' database"); + return first; + } + return defaultStorage; + } + + /** + * Returns the value of the requested variable. + *

+ * In case of list variables, the returned map is unmodifiable view of the variables map. *

- * Do not modify the returned value! + * If map is returned, it is sorted using a comparator that matches the variable name sorting. *

- * This does not take into consideration default variables. You must use get methods from {@link ch.njol.skript.lang.Variable} + * If map is returned the structure is as following: + *

    + *
  • + * If value is present for the variable and + *
      + *
    • the variable has no children, it is mapped directly to the key
      • + *
    • the variable has children it is mapped to a map, that maps {@code null} to its value and its + * children are mapped using the same strategy
      • + *
    + *
    • + *
  • If value is not present for the variable, it is mapped to a map with its children mapped using the same + * strategy
    • + *
+ *

+ * This does not take into consideration default variables. You must use get methods from {@link Variable} * - * @param name the variable's name. - * @param event if {@code local} is {@code true}, this is the event - * the local variable resides in. - * @param local if this variable is a local or global variable. - * @return an {@link Object} for a normal variable + * @param name The name of the variable. + * @param event If {@code local} is {@code true}, this is the event the local variable resides in. + * @param local If this variable is a local or global variable + * @return an {@link Object} for a regular variable * or a {@code Map} for a list variable, * or {@code null} if the variable is not set. */ - // TODO don't expose the internal value, bad API - @Nullable - public static Object getVariable(String name, @Nullable Event event, boolean local) { - String n; - if (caseInsensitiveVariables) { - n = name.toLowerCase(Locale.ENGLISH); - } else { - n = name; - } + public static @Nullable Object getVariable(String name, @Nullable Event event, boolean local) { + String fixedName = caseInsensitiveVariables + ? name.toLowerCase(Locale.ENGLISH) + : name; - if (local) { - VariablesMap map = localVariables.get(event); - if (map == null) - return null; + if (local) + return getLocalVariables(event).getVariable(fixedName); - return map.getVariable(n); - } else { - try { - variablesLock.readLock().lock(); - // Prevent race conditions from returning variables with incorrect values - if (!changeQueue.isEmpty()) { - // Gets the last VariableChange made - VariableChange variableChange = changeQueue.stream() - .filter(change -> change.name.equals(n)) - .reduce((first, second) -> second) - // Gets last value, as iteration is from head to tail, - // and adding occurs at the tail (and we want the most recently added) - .orElse(null); - - if (variableChange != null) { - return variableChange.value; - } - } - - return variables.getVariable(n); - } finally { - variablesLock.readLock().unlock(); - } - } + //noinspection resource + VariableStorage foundStorage = findStorage(fixedName); + return foundStorage.getVariable(fixedName); } /** * Returns an iterator over the values of this list variable. * * @param name the variable's name. This must be the name of a list variable, ie. it must end in *. - * @param event if {@code local} is {@code true}, this is the event - * the local variable resides in. + * @param event if {@code local} is {@code true}, this is the event the local variable resides in. * @param local if this variable is a local or global variable. - * @return an {@link Iterator} of {@link Pair}s, containing the {@link String} index and {@link Object} value of the - * elements of the list. An empty iterator is returned if the variable does not exist. + * @return an {@link Iterator} of {@link KeyedValue}, containing the {@link String} index and {@link Object} value + * of the elements of the list. An empty iterator is returned if the variable does not exist. */ - public static Iterator> getVariableIterator(String name, boolean local, @Nullable Event event) { - assert name.endsWith("*"); + public static Iterator> getVariableIterator(String name, boolean local, @Nullable Event event) { + assert name.endsWith(Variable.SEPARATOR + "*"); Object val = getVariable(name, event, local); String subName = StringUtils.substring(name, 0, -1); - if (val == null) return new EmptyIterator<>(); - assert val instanceof TreeMap; - // temporary list to prevent CMEs - @SuppressWarnings("unchecked") - Iterator keys = new ArrayList<>(((Map) val).keySet()).iterator(); + if (!(val instanceof Map map)) + throw new SkriptAPIException("Expected list variable"); + + Iterator keys = map.keySet().stream().map(String.class::cast).iterator(); + return new Iterator<>() { - @Nullable - private String key; - @Nullable - private Object next = null; + + private @Nullable String key; + private @Nullable Object next = null; @Override public boolean hasNext() { @@ -514,30 +467,32 @@ public boolean hasNext() { return true; while (keys.hasNext()) { key = keys.next(); - if (key != null) { - next = Variable.convertIfOldPlayer(subName + key, local, event, Variables.getVariable(subName + key, event, local)); - if (next != null && !(next instanceof TreeMap)) - return true; - } + if (key == null) + continue; + next = Variable.convertIfOldPlayer(subName + key, local, event, + Variables.getVariable(subName + key, event, local)); + if (next != null && !(next instanceof TreeMap)) + return true; } next = null; return false; } @Override - public Pair next() { + public KeyedValue next() { if (!hasNext()) throw new NoSuchElementException(); - Pair n = new Pair<>(key, next); + assert key != null; + assert next != null; + KeyedValue value = new KeyedValue<>(key, next); next = null; - return n; + return value; } @Override public void remove() { - if (key == null) - throw new IllegalStateException(); - Variables.deleteVariable(key, event, local); + Preconditions.checkState(key != null, "Expected non null key"); + Variables.deleteVariable(subName + key, event, local); } }; } @@ -546,8 +501,7 @@ public void remove() { * Deletes a variable. * * @param name the variable's name. - * @param event if {@code local} is {@code true}, this is the event - * the local variable resides in. + * @param event if {@code local} is {@code true}, this is the event the local variable resides in. * @param local if this variable is a local or global variable. */ public static void deleteVariable(String name, @Nullable Event event, boolean local) { @@ -557,410 +511,85 @@ public static void deleteVariable(String name, @Nullable Event event, boolean lo /** * Sets a variable. * - * @param name the variable's name. - * Can be a "list variable::*", but {@code value} - * must be {@code null} in this case. - * @param value The variable's value. Use {@code null} - * to delete the variable. - * @param event if {@code local} is {@code true}, this is the event - * the local variable resides in. + * @param name the variable's name. Can be a "list variable::*",but {@code value} must be {@code null} in this case. + * @param value The variable's value. Use {@code null} to delete the variable. + * @param event if {@code local} is {@code true}, this is the event the local variable resides in. * @param local if this variable is a local or global variable. */ public static void setVariable(String name, @Nullable Object value, @Nullable Event event, boolean local) { - if (caseInsensitiveVariables) { - name = name.toLowerCase(Locale.ENGLISH); - } - - // Check if conversion is needed due to ClassInfo#getSerializeAs - if (value != null) { - assert !name.endsWith("::*"); - - ClassInfo ci = Classes.getSuperClassInfo(value.getClass()); - Class sas = ci.getSerializeAs(); - - if (sas != null) { - value = Converters.convert(value, sas); - assert value != null : ci + ", " + sas; - } - } + String fixedName = caseInsensitiveVariables + ? name.toLowerCase(Locale.ENGLISH) + : name; if (local) { - assert event != null : name; + assert event != null : fixedName; // Get the variables map and set the variable in it - VariablesMap map = localVariables.computeIfAbsent(event, e -> new VariablesMap()); - map.setVariable(name, value); - } else { - setVariable(name, value); - } - } - - /** - * Sets the given global variable name to the given value. - * - * @param name the variable name. - * @param value the value, or {@code null} to delete the variable. - */ - static void setVariable(String name, @Nullable Object value) { - if (variablesLock.writeLock().tryLock()) { - try { - if (!changeQueue.isEmpty()) { // Process older, queued changes if available - processChangeQueue(); - } - // Process and save requested change - variables.setVariable(name, value); - saveVariableChange(name, value); - } finally { - variablesLock.writeLock().unlock(); - } - } else { - // Couldn't acquire variable write lock, queue the change (blocking here is a bad idea) - queueVariableChange(name, value); - } - } - - /** - * Changes to variables that have not yet been performed. - */ - static final Queue changeQueue = new ConcurrentLinkedQueue<>(); - - /** - * A variable change name-value pair. - */ - private static class VariableChange { - - /** - * The name of the changed variable. - */ - public final String name; - - /** - * The (possibly {@code null}) value of the variable change. - */ - @Nullable - public final Object value; - - /** - * Creates a new {@link VariableChange} with the given name and value. - * - * @param name the variable name. - * @param value the new variable value. - */ - public VariableChange(String name, @Nullable Object value) { - this.name = name; - this.value = value; - } - - } - - /** - * Queues a variable change. Only to be called when direct write is not - * possible, but thread cannot be allowed to block. - * - * @param name the variable name. - * @param value the new value. - */ - private static void queueVariableChange(String name, @Nullable Object value) { - changeQueue.add(new VariableChange(name, value)); - } - - /** - * Processes all entries in variable change queue. - *

- * Note that caller must acquire write lock before calling this, - * then release it. - */ - static void processChangeQueue() { - while (true) { // Run as long as we still have changes - VariableChange change = changeQueue.poll(); - if (change == null) - break; - - // Set and save variable - variables.setVariable(change.name, change.value); - saveVariableChange(change.name, change.value); - } - } - - /** - * Stores loaded variables while variable storages are being loaded. - *

- * Access must be synchronised. - */ - private static final SynchronizedReference>> TEMP_VARIABLES = - new SynchronizedReference<>(new HashMap<>()); - - /** - * The amount of variable conflicts between variable storages where - * a warning will be given, with any conflicts than this value, no more - * warnings will be given. - * - * @see #loadConflicts - */ - private static final int MAX_CONFLICT_WARNINGS = 50; - - /** - * Keeps track of the amount of variable conflicts between variable storages - * while loading. - */ - private static int loadConflicts = 0; - - /** - * Sets a variable and moves it to the appropriate database - * if the config was changed. - *

- * Must only be used while variables are loaded - * when Skript is starting. Must be called on Bukkit's main thread. - * This method directly invokes - * {@link VariablesStorage#save(String, String, byte[])}, - * i.e. you should not be holding any database locks or such - * when calling this! - * - * @param name the variable name. - * @param value the variable value. - * @param source the storage the variable came from. - * @return Whether the variable was stored somewhere. Not valid while storages are loading. - */ - static boolean variableLoaded(String name, @Nullable Object value, VariablesStorage source) { - assert Bukkit.isPrimaryThread(); // required by serialisation - - if (value == null) - return false; - - synchronized (TEMP_VARIABLES) { - Map> tvs = TEMP_VARIABLES.get(); - if (tvs != null) { - NonNullPair existingVariable = tvs.get(name); - - // Check for conflicts with other storages - conflict: if (existingVariable != null) { - VariablesStorage existingVariableStorage = existingVariable.getSecond(); - - if (existingVariableStorage == source) { - // No conflict if from the same storage - break conflict; - } - - // Variable already loaded from another database, conflict - loadConflicts++; - - // Warn if needed - if (loadConflicts <= MAX_CONFLICT_WARNINGS) { - Skript.warning("The variable {" + name + "} was loaded twice from different databases (" + - existingVariableStorage.getUserConfigurationName() + " and " + source.getUserConfigurationName() + - "), only the one from " + source.getUserConfigurationName() + " will be kept."); - } else if (loadConflicts == MAX_CONFLICT_WARNINGS + 1) { - Skript.warning("[!] More than " + MAX_CONFLICT_WARNINGS + - " variables were loaded more than once from different databases, " + - "no more warnings will be printed."); - } - - // Remove the value from the existing variable's storage - existingVariableStorage.save(name, null, null); - } - - // Add to the loaded variables - tvs.put(name, new NonNullPair<>(value, source)); - - return false; - } - } - - variablesLock.writeLock().lock(); - try { - variables.setVariable(name, value); - } finally { - variablesLock.writeLock().unlock(); - } - - // Move the variable to the right storage - try { - for (VariablesStorage variablesStorage : STORAGES) { - if (variablesStorage.accept(name)) { - if (variablesStorage != source) { - // Serialize and set value in new storage - Value serializedValue = serialize(value); - if (serializedValue == null) { - variablesStorage.save(name, null, null); - } else { - variablesStorage.save(name, serializedValue.type, serializedValue.data); - } - - // Remove from old storage - if (value != null) - source.save(name, null, null); - } - return true; - } - } - } catch (Exception e) { - //noinspection ThrowableNotThrown - Skript.exception(e, "Error saving variable named " + name); + getLocalVariables(event).setVariable(fixedName, value); + return; } - return false; + //noinspection resource + VariableStorage foundStorage = findStorage(fixedName); + foundStorage.setVariable(fixedName, value); } /** - * Stores loaded variables into the variables map - * and the appropriate databases. - * - * @return the amount of variables - * that don't have a storage that accepts them. + * Closes all loaded variable storages. */ - @SuppressWarnings("null") - private static int onStoragesLoaded() { - if (loadConflicts > MAX_CONFLICT_WARNINGS) - Skript.warning("A total of " + loadConflicts + " variables were loaded more than once from different databases"); - - Skript.debug("Databases loaded, setting variables..."); - - synchronized (TEMP_VARIABLES) { - Map> tvs = TEMP_VARIABLES.get(); - TEMP_VARIABLES.set(null); - assert tvs != null; - - variablesLock.writeLock().lock(); - try { - // Calculate the amount of variables that don't have a storage - int unstoredVariables = 0; - for (Entry> tv : tvs.entrySet()) { - if (!variableLoaded(tv.getKey(), tv.getValue().getFirst(), tv.getValue().getSecond())) - unstoredVariables++; - } - - for (VariablesStorage variablesStorage : STORAGES) - variablesStorage.allLoaded(); - - Skript.debug("Variables set. Queue size = " + saveQueue.size()); - - return unstoredVariables; - } finally { - variablesLock.writeLock().unlock(); - } - } + public static void close() { + STORAGES.forEach(VariableStorage::close); } /** - * Creates a {@link SerializedVariable} from the given variable name - * and value. + * Returns the number of currently loaded variables in memory. *

- * Must be called from Bukkit's main thread. + * This number may not be fully accurate. * - * @param name the variable name. - * @param value the value. - * @return the serialized variable. + * @return number of loaded variables */ - public static SerializedVariable serialize(String name, @Nullable Object value) { - assert Bukkit.isPrimaryThread(); - - // First, serialize the variable. - SerializedVariable.Value var; - try { - var = serialize(value); - } catch (Exception e) { - throw Skript.exception(e, "Error saving variable named " + name); - } - - return new SerializedVariable(name, var); + public static long numVariables() { + return STORAGES.stream() + .mapToLong(VariableStorage::loadedVariables) + .reduce(0, Long::sum); } - /** - * Serializes the given value. - *

- * Must be called from Bukkit's main thread. - * - * @param value the value to serialize. - * @return the serialized value. - */ - public static SerializedVariable.@Nullable Value serialize(@Nullable Object value) { - assert Bukkit.isPrimaryThread(); - - return Classes.serialize(value); + private Variables() { + throw new UnsupportedOperationException(); } - /** - * Serializes and adds the variable change to the {@link #saveQueue}. - * - * @param name the variable name. - * @param value the value of the variable. - */ - private static void saveVariableChange(String name, @Nullable Object value) { - if (name.startsWith(Variable.EPHEMERAL_VARIABLE_TOKEN)) - return; - saveQueue.add(serialize(name, value)); - } - - /** - * The queue of serialized variables that have not yet been written - * to the storage. - */ - static final BlockingQueue saveQueue = new LinkedBlockingQueue<>(); + private static final class BukkitConfigurationSerializer extends ConfigurationSerializer { - /** - * Whether the {@link #saveThread} should be stopped. - */ - private static volatile boolean closed = false; + /** + * The {@link ClassResolver#getID(Class) ID} prefix + * for {@link ConfigurationSerializable} classes. + */ + static final String CONFIGURATION_SERIALIZABLE_PREFIX = "ConfigurationSerializable_"; - /** - * The thread that saves variables, i.e. stores in the appropriate storage. - */ - private static final Thread saveThread = Skript.newThread(() -> { - while (!closed) { - try { - // Save one variable change - SerializedVariable variable = saveQueue.take(); - - for (VariablesStorage variablesStorage : STORAGES) { - if (variablesStorage.accept(variable.name)) { - variablesStorage.save(variable); - - break; - } - } - } catch (InterruptedException ignored) {} + BukkitConfigurationSerializer() { + // Info field is mostly unused in superclass, due to methods overridden below, + // so this illegal cast is fine + //noinspection unchecked + info = (ClassInfo) (ClassInfo) Classes.getExactClassInfo(Object.class); } - }, "Skript variable save thread"); - /** - * Closes the variable systems: - *

    - *
  • Process all changes left in the {@link #changeQueue}.
    • - *
  • Stops the {@link #saveThread}.
    • - *
- */ - public static void close() { - try { // Ensure that all changes are to save soon - variablesLock.writeLock().lock(); - processChangeQueue(); - } finally { - variablesLock.writeLock().unlock(); + @Override + public @Nullable String getID(@NotNull Class c) { + if (ConfigurationSerializable.class.isAssignableFrom(c) + && Classes.getSuperClassInfo(c) == Classes.getExactClassInfo(Object.class)) + return CONFIGURATION_SERIALIZABLE_PREFIX + + ConfigurationSerialization.getAlias(c.asSubclass(ConfigurationSerializable.class)); + return null; } - // First, make sure all variables are saved - while (saveQueue.size() > 0) { - try { - Thread.sleep(10); - } catch (InterruptedException ignored) {} + @Override + public @Nullable Class getClass(@NotNull String id) { + if (id.startsWith(CONFIGURATION_SERIALIZABLE_PREFIX)) + return ConfigurationSerialization.getClassByAlias( + id.substring(CONFIGURATION_SERIALIZABLE_PREFIX.length())); + return null; } - // Then we can safely interrupt and stop the thread - closed = true; - saveThread.interrupt(); - } - - /** - * Gets the amount of variables currently on the server. - * - * @return the amount of variables. - */ - public static int numVariables() { - try { - variablesLock.readLock().lock(); - return variables.hashMap.size(); - } finally { - variablesLock.readLock().unlock(); - } } } diff --git a/src/main/java/ch/njol/skript/variables/VariablesMap.java b/src/main/java/ch/njol/skript/variables/VariablesMap.java index 9934f46eba5..7c65c0a1b43 100644 --- a/src/main/java/ch/njol/skript/variables/VariablesMap.java +++ b/src/main/java/ch/njol/skript/variables/VariablesMap.java @@ -2,26 +2,35 @@ import ch.njol.skript.lang.Variable; import ch.njol.util.StringUtils; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import com.google.errorprone.annotations.ThreadSafe; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; +import org.jetbrains.annotations.Unmodifiable; + +import java.util.*; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.StampedLock; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.UnaryOperator; /** - * A map for storing variables in a sorted and efficient manner. + * A thread-safe Radix Tree for storing variables. */ -final class VariablesMap { +@ThreadSafe +public final class VariablesMap { /** - * The comparator for comparing variable names. + * Comparator for variable names. */ - static final Comparator VARIABLE_NAME_COMPARATOR = (s1, s2) -> { + public static final Comparator VARIABLE_NAME_COMP = (s1, s2) -> { if (s1 == null) return s2 == null ? 0 : -1; - if (s2 == null) return 1; @@ -30,76 +39,76 @@ final class VariablesMap { boolean lastNumberNegative = false; boolean afterDecimalPoint = false; + while (i < s1.length() && j < s2.length()) { char c1 = s1.charAt(i); char c2 = s2.charAt(j); - if ('0' <= c1 && c1 <= '9' && '0' <= c2 && c2 <= '9') { - // Numbers/digits are treated differently from other characters. + // Numbers/digits are treated differently from other characters. + if (Character.isDigit(c1) && Character.isDigit(c2)) { // The index after the last digit - int i2 = StringUtils.findLastDigit(s1, i); - int j2 = StringUtils.findLastDigit(s2, j); + int end1 = StringUtils.findLastDigit(s1, i); + int end2 = StringUtils.findLastDigit(s2, j); // Amount of leading zeroes - int z1 = 0; - int z2 = 0; + int leadingZeros1 = 0; + int leadingZeros2 = 0; - // Skip leading zeroes (except for the last if all 0's) if (!afterDecimalPoint) { - if (c1 == '0') { - while (i < i2 - 1 && s1.charAt(i) == '0') { - i++; - z1++; - } + while (i < end1 - 1 && s1.charAt(i) == '0') { + i++; + leadingZeros1++; } - if (c2 == '0') { - while (j < j2 - 1 && s2.charAt(j) == '0') { - j++; - z2++; - } + while (j < end2 - 1 && s2.charAt(j) == '0') { + j++; + leadingZeros2++; } } - // Keep in mind that c1 and c2 may not have the right value (e.g. s1.charAt(i)) for the rest of this block // If the number is prefixed by a '-', it should be treated as negative, thus inverting the order. // If the previous number was negative, and the only thing separating them was a '.', // then this number should also be in inverted order. - boolean previousNegative = lastNumberNegative; + int startOfNumber = i - leadingZeros1; + boolean currentIsNegative = startOfNumber > 0 && s1.charAt(startOfNumber - 1) == '-'; - // i - z1 contains the first digit, so i - z1 - 1 may contain a `-` indicating this number is negative - lastNumberNegative = i - z1 > 0 && s1.charAt(i - z1 - 1) == '-'; - int isPositive = (lastNumberNegative | previousNegative) ? -1 : 1; + // if the previous number was negative and we just crossed a dot, we stay negative + boolean effectiveNegative = currentIsNegative || lastNumberNegative; + int sign = effectiveNegative ? -1 : 1; + + int length1 = end1 - i; + int length2 = end2 - j; // Different length numbers (99 > 9) - if (!afterDecimalPoint && i2 - i != j2 - j) - return ((i2 - i) - (j2 - j)) * isPositive; + if (!afterDecimalPoint && length1 != length2) + return (length1 - length2) * sign; // Iterate over the digits - while (i < i2 && j < j2) { - char d1 = s1.charAt(i); - char d2 = s2.charAt(j); - - // If the digits differ, return a value dependent on the sign - if (d1 != d2) - return (d1 - d2) * isPositive; - + while (i < end1 && j < end2) { + int diff = s1.charAt(i) - s2.charAt(j); + if (diff != 0) + return diff * sign; i++; j++; } // Different length numbers (1.99 > 1.9) - if (afterDecimalPoint && i2 - i != j2 - j) - return ((i2 - i) - (j2 - j)) * isPositive; + if (afterDecimalPoint && length1 != length2) + return (length1 - length2) * sign; // If the numbers are equal, but either has leading zeroes, // more leading zeroes is a lesser number (01 < 1) - if (z1 != z2) - return (z1 - z2) * isPositive; + if (leadingZeros1 != leadingZeros2) + return (leadingZeros1 - leadingZeros2) * sign; + // We finished processing a number, we are now "after" a number. + // If the next char is a dot, we remain in decimal mode. afterDecimalPoint = true; - } else { - // Normal characters + // this is for backwards compatibility, else it should be effectiveNegative + lastNumberNegative = currentIsNegative; + } + // Normal characters + else { if (c1 != c2) return c1 - c2; @@ -113,6 +122,8 @@ final class VariablesMap { j++; } } + + // One is prefix of the other if (i < s1.length()) return lastNumberNegative ? -1 : 1; if (j < s2.length()) @@ -121,60 +132,253 @@ final class VariablesMap { }; /** - * The map that stores all non-list variables. + * A node in the radix tree. + *

+ * This also serves as a thread safe unmodifiable live view of the tree branch branch in + * the format returned by {@link #getVariable(String)}. + *

+ * It does not lock the tree and is weakly consistent, modifications to the underlying + * variables map by other threads may not be immediately visible, prioritizing performance + * over strict in time snapshots. There is no way to verify whether the node is still + * valid (part of the tree). */ - final HashMap hashMap = new HashMap<>(); + private static class Node extends AbstractMap { + + /** + * Lock that is read locked when entering to the node + * and released when the operation that locked it, is complete. + *

+ * Write lock is acquired only when clearing an entire subtree + * of a node to ensure all operations on that subtree are + * completed before it is cleared or when calling {@link #prune()} + * which needs to lock the entire tree. + */ + final StampedLock lock = new StampedLock(); + + /** + * Current value assigned to this variable or {@code null} if + * no value is set. + */ + final AtomicReference<@Nullable Object> ref = new AtomicReference<>(); + + /** + * Children of this node. + *

+ * This uses {@link ConcurrentSkipListMap} because: + *
+ *

  • The iterator used by the live-view of this node must be thread safe
    • + *
  • The map itself must be thread safe as it is modified under read lock of the node
    • + *
  • The children need to be sorted by the variables name compare
    • + *
    + * This allows us to use the live view of this map as a view of this node + * (if transformed to match the map format of {@link #getVariable(String)}). + */ + final Map children = new ConcurrentSkipListMap<>(VARIABLE_NAME_COMP); + + /** + * @return whether the node has children + */ + boolean hasChildren() { + return !children.isEmpty(); + } + + /** + * @return whether the node is empty (has no value and no children) + */ + @Override + public boolean isEmpty() { + return ref.get() == null && !hasChildren(); + } + + @Override + public int size() { + int size = children.size(); + return ref.get() != null ? ++size : size; // include the value if present as it is mapped to null key + } + + @Override + public boolean containsKey(Object key) { + return get(key) != null; + } + + @Override + public Object get(Object key) { + if (key == null) + return ref.get(); + Node child = children.get(key); + return child != null ? child.unwrap() : null; + } + + @Override + public @NotNull Set> entrySet() { + return new AbstractSet<>() { + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public @NotNull Iterator iterator() { + Object value = Node.this.ref.get(); + + Iterator> wrapped = children.entrySet().iterator(); + Iterator> iterator; + + if (value != null) { + // concat iterator with the value of this node if present + Iterator> itself = + (Iterator) Collections.singleton(new SimpleEntry<>(null, value)).iterator(); + // source iterators are not polled until necessary, the null key is first + iterator = Iterators.concat(itself, (Iterator) wrapped); + } else { + iterator = (Iterator) wrapped; + } + + // this transformation is lazy + return Iterators.transform(iterator, entry -> { + if (entry.getKey() != null /* sub tree */) { + Node node = (Node) entry.getValue(); + return new SimpleEntry<>(entry.getKey(), node.unwrap()); + } else { + return entry; // null key with value of this node + } + }); + } + + @Override + public int size() { + return Node.this.size(); + } + }; + } + + /** + * @return returns the representation of this node in the exposed map + */ + private Object unwrap() { + return hasChildren() ? this : ref.get(); + } + + } + /** - * The tree of variables, branched by the list structure of the variables. + * Root node of the tree. */ - final TreeMap treeMap = new TreeMap<>(); + private final Node root = new Node(); /** - * Returns the internal value of the requested variable. + * Estimate of empty branches in the radix tree. *

    - * Do not modify the returned value! + * The real number may be different as some branches may be re-populated after clear. + */ + private final AtomicInteger leftEmpty = new AtomicInteger(0); + + /** + * At how many writes that leave empty branches {@link #prune()} should be executed. + */ + private final int pruneAt; + + /** + * Executor of automatic prune operation. + */ + private final Executor pruneExecutor; + + /** + * Constructs new variables map that automatically calls {@link #prune()} + * after certain number of {@link #setVariable(String, Object)} left + * empty branches in the radix tree. + * + * @param pruneAt after which number of such writes the variables map should call prune + * @param pruneExecutor executor which will execute the expensive prune operation + */ + public VariablesMap(int pruneAt, Executor pruneExecutor) { + this.pruneAt = pruneAt; + this.pruneExecutor = pruneExecutor; + } + + /** + * Constructs new variables map that automatically calls {@link #prune()} + * after certain number of {@link #setVariable(String, Object)} left + * empty branches in the radix tree. + * + * @param pruneAt after which number of such writes the variables map should call prune + */ + public VariablesMap(int pruneAt) { + this(pruneAt, Runnable::run); + } + + /** + * Constructs new variables map. + */ + public VariablesMap() { + this(Integer.MAX_VALUE, Runnable::run); + } + + /** + * Returns the value of the requested variable. + *

    + * In case of list variables, the returned map is thread safe unmodifiable live view of the variables map. + *

    + * If map is returned, it is sorted using the variables name comparator. + *

    + * If map is returned the structure is as following: + *

      + *
    • + * If value is present for the variable and + *
        + *
      • the variable has no children, its value is mapped directly to the key
        • + *
      • the variable has children, it is mapped to a map, that maps {@code null} to its value and its + * children are mapped using the same strategy
        • + *
      + *
      • + *
    • If value is not present for the variable, it is mapped to a map with its children mapped using the same + * strategy
      • + *
    * * @param name the name of the variable, possibly a list variable. * @return an {@link Object} for a normal variable or a * {@code Map} for a list variable, * or {@code null} if the variable is not set. */ - @SuppressWarnings("unchecked") - @Nullable - Object getVariable(String name) { - if (!name.endsWith("*")) { - // Not a list variable, quick access from the hash map - return hashMap.get(name); - } else { - // List variable, search the tree branches - String[] split = Variables.splitVariableName(name); - Map parent = treeMap; - - // Iterate over the parts of the variable name - for (int i = 0; i < split.length; i++) { - String n = split[i]; - if (n.equals("*")) { - // End of variable name, return map - assert i == split.length - 1; - return parent; - } + public @Nullable Object getVariable(String name) { + boolean isList = name.endsWith(Variable.SEPARATOR + "*"); + if (isList) + name = name.substring(0, name.length() - (Variable.SEPARATOR.length() + 1)); // strip the "::*" suffix - // Check if the current (sub-)tree has the expected child node - Object childNode = parent.get(n); - if (childNode == null) - return null; + String[] parts = Variables.splitVariableName(name); + + int limit = parts.length + 1; // +1 for the root node + Node[] path = new Node[limit]; + long[] stamps = new long[limit]; + int depth = 0; + + Node current = root; + path[0] = current; + stamps[0] = current.lock.readLock(); - // Continue the iteration if the child node is a tree itself - if (childNode instanceof Map) { - // Continue iterating with the subtree - parent = (Map) childNode; - assert i != split.length - 1; - } else { - // ..., otherwise the list variable doesn't exist here + try { + for (String part : parts) { + if (!current.hasChildren()) return null; - } + Node next = current.children.get(part); + if (next == null) + return null; + + long nextStamp = next.lock.readLock(); + + depth++; + path[depth] = next; + stamps[depth] = nextStamp; + + current = next; + } + + if (isList) { + return current; + } else { + return current.ref.get(); + } + } finally { + for (int i = depth; i >= 0; i--) { + if (path[i] != null) + path[i].lock.unlockRead(stamps[i]); } - return null; } } @@ -186,124 +390,186 @@ Object getVariable(String name) { * * @param name the variable name. * @param value the variable value, {@code null} to delete the variable. + * @return previous value for changed variable, {@code null} if not set or + * the variable is a list that was cleared */ - @SuppressWarnings("unchecked") - void setVariable(String name, @Nullable Object value) { - // First update the hash map easily - if (!name.endsWith("*")) { - if (value == null) - hashMap.remove(name); - else - hashMap.put(name, value); + public @Nullable Object setVariable(String name, @Nullable Object value) { + boolean isList = name.endsWith(Variable.SEPARATOR + "*"); + + String actualName = isList + ? name.substring(0, name.length() - (Variable.SEPARATOR.length() + 1)) + : name; + + if (isList) { + Preconditions.checkState(value == null, "List variables can only be set to null"); } - // Then update the tree map by going down the branches - String[] split = Variables.splitVariableName(name); - TreeMap parent = treeMap; - - // Iterate over the parts of the variable name - for (int i = 0; i < split.length; i++) { - String childNodeName = split[i]; - Object childNode = parent.get(childNodeName); - - if (childNode == null) { - // Expected child node not found - if (i == split.length - 1) { - // End of the variable name reached, set variable if needed - if (value != null) - parent.put(childNodeName, value); - - break; - } else if (value != null) { - // Create child node, add it to parent and continue iteration - childNode = new TreeMap<>(VARIABLE_NAME_COMPARATOR); - - parent.put(childNodeName, childNode); - parent = (TreeMap) childNode; - } else { - // Want to set variable to null, bu variable is already null - break; - } - } else if (childNode instanceof TreeMap) { - // Child node found - TreeMap childNodeMap = ((TreeMap) childNode); - - if (i == split.length - 1) { - // End of variable name reached, adjust child node accordingly - if (value == null) - childNodeMap.remove(null); - else - childNodeMap.put(null, value); - - break; - } else if (i == split.length - 2 && split[i + 1].equals("*")) { - // Second to last part of variable name - assert value == null; - - // Delete all indices of the list variable from hashMap - deleteFromHashMap(StringUtils.join(split, Variable.SEPARATOR, 0, i + 1), childNodeMap); - - // If the list variable itself has a value , - // e.g. list `{mylist::3}` while variable `{mylist}` also has a value, - // then adjust the parent for that - Object currentChildValue = childNodeMap.get(null); - if (currentChildValue == null) - parent.remove(childNodeName); - else - parent.put(childNodeName, currentChildValue); - - break; - } else { - // Continue iteration - parent = childNodeMap; - } - } else { - // Ran into leaf node - if (i == split.length - 1) { - // If we arrived at the end of the variable name, update parent - if (value == null) - parent.remove(childNodeName); - else - parent.put(childNodeName, value); - - break; - } else if (value != null) { - // Need to continue iteration, create new child node and put old value in it - TreeMap newChildNodeMap = new TreeMap<>(VARIABLE_NAME_COMPARATOR); - newChildNodeMap.put(null, childNode); - - // Add new child node to parent - parent.put(childNodeName, newChildNodeMap); - parent = newChildNodeMap; - } else { - break; - } + String[] parts = Variables.splitVariableName(actualName); + + if (isList) { // we are clearing a list + clearListVariable(root, parts); + return null; + } else { + return modifySingleVariable(root, parts, old -> value /* discard previous, set to new */); + } + } + + /** + * Returns the variable with given name and if there is none set, sets it to + * the next value provided by the mapping function. + *

    + * This method only accepts single variables. + *

    + * The {@code mappingFunction} is executed under a write lock on the variable's node. + * Do not perform expensive operations or access other variables inside this function to avoid + * deadlock and performance degradation. + * + * @param name the variable name. + * @param mappingFunction function providing the new value in case it is not set + * @return current value of the variable + */ + public Object computeIfAbsent(String name, Function mappingFunction) { + Preconditions.checkState(!name.endsWith(Variable.SEPARATOR + "*")); + AtomicReference got = new AtomicReference<>(); + String[] parts = Variables.splitVariableName(name); + modifySingleVariable(root, parts, prev -> { + if (prev == null) { + Object computed = mappingFunction.apply(name); + got.set(computed); + return computed; } + got.set(prev); + return prev; + }); + return got.get(); + } + + /** + * Applies operation at node of given variable under its write lock. + * + * @param root root node + * @param parts parts of the variable + * @param operation operation to apply + * @return value associated with the node before the operation + */ + private @Nullable Object modifySingleVariable(Node root, String[] parts, UnaryOperator operation) { + int limit = parts.length + 1; // +1 for the root node + Node[] path = new Node[limit]; + long[] stamps = new long[limit]; + int depth = 0; + + Node current = root; + path[0] = current; + stamps[0] = current.lock.readLock(); + + try { + for (String part : parts) { + // this can be done under read lock because the children map implementation + // itself is thread safe + Node next = current.children.computeIfAbsent(part, key -> new Node()); + + long nextStamp = next.lock.readLock(); + + depth++; + path[depth] = next; + stamps[depth] = nextStamp; + + current = next; + } + + Object prev = current.ref.getAndUpdate(operation); + checkForPrune(current); + return prev; + } finally { + for (int i = depth; i >= 0; i--) { + if (path[i] != null) + path[i].lock.unlock(stamps[i]); + } + } + } + + /** + * Clears a list variable. + *

    + * Compare to other operations, this one functions a lot differently and + * does not need to read lock the entire path. + *

    + * This is the only operation that uses the write lock of the node it is clearing. + * Reason for this is, all operations happening on this part of the sub-tree + * also hold a read lock for this particular node, meaning it: + *

      + *
    • Stops any other future operations from happening until the list clear completes
      • + *
    • Waits for all operations happening in the sub-tree to finish
      • + *
    + * This ensures no invalid values are returned (different thread could return already deleted + * values otherwise). + * + * @param root root node + * @param parts parts of the variable to clear + */ + private void clearListVariable(Node root, String[] parts) { + Node current = root; + for (String key : parts) { + Node next = current.children.get(key); + // if child does not exist there is nothing to clear + if (next == null) + return; + current = next; + } + long stamp = current.lock.writeLock(); + try { + current.children.clear(); + } finally { + current.lock.unlockWrite(stamp); + checkForPrune(current); } } /** - * Deletes all indices of a list variable from the {@link #hashMap}. + * Checks if the node if empty, if yes, it increases the + * empty nodes counter and possibly triggers automatic {@link #prune()} call + * if the number of empty nodes exceeds {@link #pruneAt}. + *

    + * This does not have to be fully accurate and atomic with the clear operations + * themselves, as {@link #leftEmpty} is only an estimate. * - * @param parent the list variable prefix, - * e.g. {@code list} for {@code list::*}. - * @param current the map of the list variable. + * @param node node to check after appplying an operation */ - @SuppressWarnings("unchecked") - void deleteFromHashMap(String parent, TreeMap current) { - for (Entry e : current.entrySet()) { - if (e.getKey() == null) - continue; - String childName = parent + Variable.SEPARATOR + e.getKey(); - - // Remove from hashMap - hashMap.remove(childName); - - // Recurse if needed - Object val = e.getValue(); - if (val instanceof TreeMap) { - deleteFromHashMap(childName, (TreeMap) val); + private void checkForPrune(Node node) { + if (!node.isEmpty()) + return; + int count = leftEmpty.incrementAndGet(); + if (count >= pruneAt && leftEmpty.compareAndSet(count, 0)) { + pruneExecutor.execute(this::prune); + } + } + + /** + * Prunes the entire tree, removing all empty nodes. + *

    + * This operation is expensive and fully write locks the radix tree. + */ + public void prune() { + prune(root); + } + + private boolean prune(Node node) { + long stamp = node.lock.writeLock(); + try { + if (node.isEmpty()) + return true; + + var it = node.children.entrySet().iterator(); + while (it.hasNext()) { + var entry = it.next(); + boolean isChildEmpty = prune(entry.getValue()); + if (isChildEmpty) + it.remove(); } + + return node.isEmpty(); + } finally { + node.lock.unlockWrite(stamp); } } @@ -314,41 +580,87 @@ void deleteFromHashMap(String parent, TreeMap current) { */ public VariablesMap copy() { VariablesMap copy = new VariablesMap(); + copy(this.root, copy.root); + return copy; + } - copy.hashMap.putAll(hashMap); - - TreeMap treeMapCopy = copyTreeMap(treeMap); - copy.treeMap.putAll(treeMapCopy); + private void copy(Node source, Node target) { + long stamp = source.lock.readLock(); + try { + target.ref.set(source.ref.get()); + if (source.hasChildren()) { + source.children.forEach((key, sourceChild) -> { + Node targetChild = new Node(); + copy(sourceChild, targetChild); + target.children.put(key, targetChild); + }); + } + } finally { + source.lock.unlockRead(stamp); + } + } - return copy; + /** + * @return whether the variables map is empty + */ + public boolean isEmpty() { + return size() == 0; } /** - * Makes a deep copy of the given {@link TreeMap}. + * Returns all variables in this map. + *

    + * The map is unmodifiable and ordered in the variables name order. *

    - * The 'deep copy' means that each subtree of the given tree is copied - * as well. + * This map is not nested and contains variables in format {@code full key <-> value} * - * @param original the original tree map. - * @return the copy. + * @return all variables in this map */ - @SuppressWarnings("unchecked") - private static TreeMap copyTreeMap(TreeMap original) { - TreeMap copy = new TreeMap<>(VARIABLE_NAME_COMPARATOR); - - for (Entry child : original.entrySet()) { - String key = child.getKey(); - Object value = child.getValue(); + public @Unmodifiable Map getAll() { + Map all = new TreeMap<>(VARIABLE_NAME_COMP); + getAll("", root, all::put); + return Collections.unmodifiableMap(all); + } - // Copy by recursion if the child is a TreeMap - if (value instanceof TreeMap) { - value = copyTreeMap((TreeMap) value); + private void getAll(String buffer, Node source, BiConsumer collector) { + long stamp = source.lock.readLock(); + try { + if (source.ref.get() != null) + collector.accept(buffer, source.ref.get()); + if (source.hasChildren()) { + source.children.forEach((key, child) -> { + String nextName = buffer.isEmpty() ? key : buffer + Variable.SEPARATOR + key; + getAll(nextName, child, collector); + }); } - - copy.put(key, value); + } finally { + source.lock.unlockRead(stamp); } + } - return copy; + /** + * Returns number of variables in this map. + * + * @return number of variables in this map + */ + public long size() { + return size(root); + } + + private long size(Node node) { + long stamp = node.lock.readLock(); + long size = 0; + try { + if (node.ref.get() != null) + size++; + if (node.hasChildren()) { + for (Node child : node.children.values()) + size += size(child); + } + } finally { + node.lock.unlockRead(stamp); + } + return size; } } diff --git a/src/main/java/ch/njol/skript/variables/VariablesStorage.java b/src/main/java/ch/njol/skript/variables/VariablesStorage.java deleted file mode 100644 index 10c4bf99d01..00000000000 --- a/src/main/java/ch/njol/skript/variables/VariablesStorage.java +++ /dev/null @@ -1,512 +0,0 @@ -package ch.njol.skript.variables; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -import org.jetbrains.annotations.Nullable; - -import ch.njol.skript.Skript; -import ch.njol.skript.config.SectionNode; -import ch.njol.skript.lang.ParseContext; -import ch.njol.skript.log.ParseLogHandler; -import ch.njol.skript.log.SkriptLogger; -import ch.njol.skript.registrations.Classes; -import ch.njol.skript.util.FileUtils; -import ch.njol.skript.util.Task; -import ch.njol.skript.util.Timespan; -import ch.njol.skript.variables.SerializedVariable.Value; -import ch.njol.util.Closeable; - -/** - * A variable storage is holds the means and methods of storing variables. - *

    - * This is usually some sort of database, and could be as simply as a text file. - * - * @see FlatFileStorage - * @see DatabaseStorage - */ -// FIXME ! large databases (>25 MB) cause the server to be unresponsive instead of loading slowly -@SuppressWarnings({"SuspiciousIndentAfterControlStatement", "removal"}) -public abstract class VariablesStorage implements Closeable { - - /** - * The size of the variable changes queue. - */ - private static final int QUEUE_SIZE = 1000; - /** - * The threshold of the size of the variable change - * after which a warning will be sent. - */ - private static final int FIRST_WARNING = 300; - - final LinkedBlockingQueue changesQueue = new LinkedBlockingQueue<>(QUEUE_SIZE); - - /** - * Whether this variable storage has been {@link #close() closed}. - */ - protected volatile boolean closed = false; - - /** - * The name of the database - */ - private String databaseName; - - /** - * The type of the database, i.e. CSV. - */ - private final String databaseType; - - /** - * The file associated with this variable storage. - * Can be {@code null} if no file is required. - */ - @Nullable - protected File file; - - /** - * The pattern of the variable name this storage accepts. - * {@code null} for '{@code .*}' or '{@code .*}'. - */ - @Nullable - private Pattern variableNamePattern; - - /** - * The thread used for writing variables to the storage. - */ - // created in the constructor, started in load() - private final Thread writeThread; - - /** - * Creates a new variable storage with the given name. - *

    - * This will also create the {@link #writeThread}, but it must be started - * with {@link #load(SectionNode)}. - * - * @param type the database type i.e. CSV. - */ - protected VariablesStorage(String type) { - assert type != null; - databaseType = type; - - writeThread = Skript.newThread(() -> { - while (!closed) { - try { - // Take a variable from the queue and process it - SerializedVariable variable = changesQueue.take(); - Value value = variable.value; - - // Actually save the variable - if (value != null) - save(variable.name, value.type, value.data); - else - save(variable.name, null, null); - } catch (InterruptedException ignored) { - // Ignored as the `closed` field will indicate whether the thread actually needs to stop - } - } - }, "Skript variable save thread for database '" + type + "'"); - } - - /** - * Get the config name of a database - *

    - * Note: Returns the user set name for the database, ex: - * 

    {@code
-	 * default: <- Config Name
-	 *    type: CSV
-	 * }
    - * @return name of database - */ - protected final String getUserConfigurationName() { - return databaseName; - } - - /** - * Get the config type of a database - * @return type of databse - */ - protected final String getDatabaseType() { - return databaseType; - } - - /** - * Gets the string value at the given key of the given section node. - * - * @param sectionNode the section node. - * @param key the key. - * @return the value, or {@code null} if the value was invalid, - * or not found. - */ - @Nullable - protected String getValue(SectionNode sectionNode, String key) { - return getValue(sectionNode, key, String.class); - } - - /** - * Gets the value at the given key of the given section node, - * parsed with the given type. - * - * @param sectionNode the section node. - * @param key the key. - * @param type the type. - * @return the parsed value, or {@code null} if the value was invalid, - * or not found. - * @param the type. - */ - @Nullable - protected T getValue(SectionNode sectionNode, String key, Class type) { - String rawValue = sectionNode.getValue(key); - // Section node doesn't have this key - if (rawValue == null) { - Skript.error("The config is missing the entry for '" + key + "' in the database '" + databaseName + "'"); - return null; - } - - try (ParseLogHandler log = SkriptLogger.startParseLogHandler()) { - T parsedValue = Classes.parse(rawValue, type, ParseContext.CONFIG); - - if (parsedValue == null) - // Parsing failed - log.printError("The entry for '" + key + "' in the database '" + databaseName + "' must be " + - Classes.getSuperClassInfo(type).getName().withIndefiniteArticle()); - else - log.printLog(); - - return parsedValue; - } - } - - private static final Set registeredFiles = new HashSet<>(); - - /** - * Loads the configuration for this variable storage - * from the given section node. - * - * @param sectionNode the section node. - * @return whether the loading succeeded. - */ - public final boolean load(SectionNode sectionNode) { - databaseName = sectionNode.getKey(); - - String pattern = getValue(sectionNode, "pattern"); - if (pattern == null) - return false; - - try { - // Set variable name pattern, see field javadoc for explanation of null value - variableNamePattern = pattern.equals(".*") || pattern.equals(".+") ? null : Pattern.compile(pattern); - } catch (PatternSyntaxException e) { - Skript.error("Invalid pattern '" + pattern + "': " + e.getLocalizedMessage()); - return false; - } - - if (requiresFile()) { - // Initialize file - String fileName = getValue(sectionNode, "file"); - if (fileName == null) - return false; - - this.file = getFile(fileName).getAbsoluteFile(); - - if (file.exists() && !file.isFile()) { - Skript.error("The database file '" + file.getName() + "' must be an actual file, not a directory."); - return false; - } - - // Create the file if it does not exist yet - try { - //noinspection ResultOfMethodCallIgnored - file.createNewFile(); - } catch (IOException e) { - Skript.error("Cannot create the database file '" + file.getName() + "': " + e.getLocalizedMessage()); - return false; - } - - // Check for read & write permissions to the file - if (!file.canWrite()) { - Skript.error("Cannot write to the database file '" + file.getName() + "'!"); - return false; - } - if (!file.canRead()) { - Skript.error("Cannot read from the database file '" + file.getName() + "'!"); - return false; - } - - if (registeredFiles.contains(file)) { - Skript.error("Database `" + databaseName + "` failed to load. The file `" + fileName + "` is already registered to another database."); - return false; - } - registeredFiles.add(file); - - // Set the backup interval, if present & enabled - if (!"0".equals(getValue(sectionNode, "backup interval"))) { - Timespan backupInterval = getValue(sectionNode, "backup interval", Timespan.class); - int toKeep = getValue(sectionNode, "backups to keep", Integer.class); - boolean removeBackups = false; - boolean startBackup = true; - if (backupInterval != null) - if (toKeep == 0) { - startBackup = false; - } else if (toKeep >= 1) { - removeBackups = true; - } - if (startBackup) { - startBackupTask(backupInterval, removeBackups, toKeep); - } else { - try { - FileUtils.backupPurge(file, toKeep); - } catch (IOException e) { - Skript.error("Variables backup wipe failed: " + e.getLocalizedMessage()); - } - } - } - } - - // Load the entries custom to the variable storage - if (!load_i(sectionNode)) - return false; - - writeThread.start(); - Skript.closeOnDisable(this); - - return true; - } - - /** - * Loads variables stored here. - * - * @return Whether the database could be loaded successfully, - * i.e. whether the config is correct and all variables could be loaded. - */ - protected abstract boolean load_i(SectionNode n); - - /** - * Called after all storages have been loaded, and variables - * have been redistributed if settings have changed. - * This should commit the first transaction (which is not empty if - * variables have been moved from another database to this one or vice versa), - * and start repeating transactions if applicable. - */ - protected abstract void allLoaded(); - - /** - * Checks if this storage requires a file for storing its data. - * - * @return if this storage needs a file. - */ - protected abstract boolean requiresFile(); - - /** - * Gets the file needed for this variable storage from the given file name. - *

    - * Will only be called if {@link #requiresFile()} is {@code true}. - * - * @param fileName the given file name. - * @return the {@link File} object. - */ - protected abstract File getFile(String fileName); - - /** - * Must be locked after {@link Variables#getReadLock()} - * (if that lock is used at all). - */ - protected final Object connectionLock = new Object(); - - /** - * (Re)connects to the database. - *

    - * Not called on the first connect: do this in {@link #load_i(SectionNode)}. - * An error should be printed by this method - * prior to returning {@code false}. - * - * @return whether the connection could be re-established. - */ - protected abstract boolean connect(); - - /** - * Disconnects from the database. - */ - protected abstract void disconnect(); - - /** - * The backup task, or {@code null} if automatic backups are disabled. - */ - @Nullable - protected Task backupTask = null; - - /** - * Starts the backup task, with the given backup interval. - * - * @param backupInterval the backup interval. - */ - public void startBackupTask(Timespan backupInterval, boolean removeBackups, int toKeep) { - // File is null or backup interval is invalid - if (file == null || backupInterval.getAs(Timespan.TimePeriod.TICK) == 0) - return; - backupTask = new Task(Skript.getInstance(), backupInterval.getAs(Timespan.TimePeriod.TICK), backupInterval.getAs(Timespan.TimePeriod.TICK), true) { - @Override - public void run() { - synchronized (connectionLock) { - // Disconnect, - disconnect(); - try { - // ..., then backup - FileUtils.backup(file); - if (removeBackups) { - try { - FileUtils.backupPurge(file, toKeep); - } catch (IOException | IllegalArgumentException e) { - Skript.error("Automatic variables backup purge failed: " + e.getLocalizedMessage()); - } - } - } catch (IOException e) { - Skript.error("Automatic variables backup failed: " + e.getLocalizedMessage()); - } finally { - // ... and reconnect - connect(); - } - } - } - }; - } - - /** - * Checks if this variable storage accepts the given variable name. - * - * @param var the variable name. - * @return if this storage accepts the variable name. - * - * @see #variableNamePattern - */ - boolean accept(@Nullable String var) { - if (var == null) - return false; - - return variableNamePattern == null || variableNamePattern.matcher(var).matches(); - } - - /** - * Returns the name pattern accepted by this variable storage - * @return the name pattern, or null if accepting all - */ - public @Nullable Pattern getNamePattern() { - return variableNamePattern; - } - - /** - * The interval between warnings that many variables are being written - * at once, in seconds. - */ - private static final int WARNING_INTERVAL = 10; - /** - * The interval between errors that too many variables are being written - * at once, in seconds. - */ - private static final int ERROR_INTERVAL = 10; - - /** - * The last time a warning was printed for many variables in the queue. - */ - private long lastWarning = Long.MIN_VALUE; - /** - * The last time an error was printed for too many variables in the queue. - */ - private long lastError = Long.MIN_VALUE; - - /** - * Saves the given serialized variable. - *

    - * May be called from a different thread than Bukkit's main thread. - * - * @param var the serialized variable. - */ - final void save(SerializedVariable var) { - if (changesQueue.size() > FIRST_WARNING && lastWarning < System.currentTimeMillis() - WARNING_INTERVAL * 1000) { - // Too many variables queued up to save, warn the server - Skript.warning("Cannot write variables to the database '" + databaseName + "' at sufficient speed; " + - "server performance may suffer and many variables will be lost if the server crashes. " + - "(this warning will be repeated at most once every " + WARNING_INTERVAL + " seconds)"); - - lastWarning = System.currentTimeMillis(); - } - - if (!changesQueue.offer(var)) { - // Variable changes queue filled up - - if (lastError < System.currentTimeMillis() - ERROR_INTERVAL * 1000) { - // Inform console about overload of variable changes - Skript.error("Skript cannot save any variables to the database '" + databaseName + "'. " + - "The server will hang and may crash if no more variables can be saved."); - - lastError = System.currentTimeMillis(); - } - - // Halt thread until variables queue starts clearing up - while (true) { - try { - // REMIND add repetitive error and/or stop saving variables altogether? - changesQueue.put(var); - break; - } catch (InterruptedException ignored) {} - } - } - } - - /** - * Called when Skript gets disabled. - *

    - * The default implementation will wait for all variables to be saved - * before setting {@link #closed} to {@code true} and stopping - * the {@link #writeThread write thread}. - *

    - * Therefore, make sure to call {@code super.close()} - * if this method is overridden. - */ - @Override - public void close() { - // Wait for all variable changes to be processed - while (changesQueue.size() > 0) { - try { - Thread.sleep(10); - } catch (InterruptedException ignored) {} - } - - // Now safely close storage and interrupt thread - closed = true; - writeThread.interrupt(); - } - - /** - * Clears the {@link #changesQueue queue} of unsaved variables. - *

    - * Only used if all variables are saved immediately - * after calling this method. - */ - protected void clearChangesQueue() { - changesQueue.clear(); - } - - /** - * Saves a variable. - *

    - * This is called from the main thread - * while variables are transferred between databases, - * and from the {@link #writeThread} afterwards. - *

    - * {@code type} and {@code value} are both {@code null} - * iff this call is to delete the variable. - * - * @param name the name of the variable. - * @param type the type of the variable. - * @param value the serialized value of the variable. - * @return Whether the variable was saved. - */ - protected abstract boolean save(String name, @Nullable String type, @Nullable byte[] value); - -} diff --git a/src/main/java/org/skriptlang/skript/variables/storage/H2Storage.java b/src/main/java/org/skriptlang/skript/variables/storage/H2Storage.java new file mode 100644 index 00000000000..6e001269fa4 --- /dev/null +++ b/src/main/java/org/skriptlang/skript/variables/storage/H2Storage.java @@ -0,0 +1,135 @@ +package org.skriptlang.skript.variables.storage; + +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.JdbcStorage; +import com.zaxxer.hikari.HikariConfig; +import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.io.File; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; + +/** + * H2 storage for Skript variables. + */ +@SuppressWarnings("SqlSourceToSinkFlow") +public class H2Storage extends JdbcStorage { + + public H2Storage(SkriptAddon source, String type) { + super(source, type); + } + + @Override + protected @Nullable HikariConfig configuration(SectionNode sectionNode) { + if (file == null) + return null; + assert file.getName().endsWith(".mv.db"); + + HikariConfig configuration = new HikariConfig(); + configuration.setPoolName("H2-Pool"); + configuration.setDataSourceClassName("org.h2.jdbcx.JdbcDataSource"); + configuration.setConnectionTestQuery("VALUES 1"); + + String url = ""; + if (sectionNode.get("memory", "false").equalsIgnoreCase("true")) + url += "mem:"; + url += "file:" + file.getAbsolutePath(); + url = url.substring(0, url.length() - ".mv.db".length()); + + String h2Settings = ";COMPRESS=TRUE" + + ";RETENTION_TIME=1000" + + ";CACHE_SIZE=32768" + + ";TRACE_LEVEL_FILE=0" + + ";TRACE_LEVEL_SYSTEM_OUT=0" + + ";DB_CLOSE_ON_EXIT=FALSE"; + + configuration.addDataSourceProperty("URL", "jdbc:h2:" + url + h2Settings); + configuration.addDataSourceProperty("user", sectionNode.get("user", "")); + configuration.addDataSourceProperty("password", sectionNode.get("password", "")); + configuration.addDataSourceProperty("description", sectionNode.get("description", "")); + return configuration; + } + + @Override + protected boolean requiresFile() { + return true; + } + + @Override + protected File getFile(String fileName) { + if (!fileName.endsWith(".mv.db")) + fileName = fileName + ".mv.db"; // H2 automatically appends '.mv.db' to the file from url + return new File(fileName); + } + + // language=H2 + @Override + protected String createTableQuery() { + return "CREATE TABLE IF NOT EXISTS " + table + " (" + + "`name` VARCHAR(" + MAX_VARIABLE_NAME_LENGTH + ") NOT NULL PRIMARY KEY," + + "`type` VARCHAR(" + MAX_CLASS_CODENAME_LENGTH + ")," + + "`value` BINARY LARGE OBJECT(" + MAX_VALUE_SIZE + ")" + + ");"; + } + + @Override + protected PreparedStatement readSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("SELECT `type`, `value` FROM " + table + " WHERE `name` = ?"); + } + + @Override + protected PreparedStatement readListQuery(Connection connection) throws SQLException { + return connection.prepareStatement("SELECT `name`, `type`, `value` FROM " + table + " WHERE `name` LIKE ?"); + } + + @Override + protected PreparedStatement writeSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("MERGE INTO " + table + " (`name`, `type`, `value`) KEY(`name`) VALUES (?, ?, ?)"); + } + + @Override + protected PreparedStatement writeMultipleQuery(Connection connection) throws SQLException { + return writeSingleQuery(connection); + } + + @Override + protected PreparedStatement deleteSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement("DELETE FROM " + table + " WHERE `name` = ?"); + } + + @Override + protected PreparedStatement deleteListQuery(Connection connection) throws SQLException { + return connection.prepareStatement("DELETE FROM " + table + " WHERE `name` LIKE ?"); + } + + @Override + protected void afterSave(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CHECKPOINT"); + } + } + + @Override + protected void closeDatabase() throws SQLException { + if (database != null) { + try (Connection conn = database.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("SHUTDOWN"); + } catch (SQLException exception) { + // 90121: Database is already closed + // This happens because we just executed SHUTDOWN, so the DB just closed + // and try with resources tries to close the connection again + // We can safely ignore this. + if (exception.getErrorCode() != 90121) { + throw exception; + } + } + if (!database.isClosed()) + database.close(); + } + } + +} diff --git a/src/main/java/org/skriptlang/skript/variables/storage/InMemoryVariableStorage.java b/src/main/java/org/skriptlang/skript/variables/storage/InMemoryVariableStorage.java new file mode 100644 index 00000000000..942f35734bc --- /dev/null +++ b/src/main/java/org/skriptlang/skript/variables/storage/InMemoryVariableStorage.java @@ -0,0 +1,59 @@ +package org.skriptlang.skript.variables.storage; + +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.VariableStorage; +import ch.njol.skript.variables.VariablesMap; +import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.io.File; + +/** + * Variable storage that stores variables in heap memory and + * discards them on close. + *

    + * This implementation of storage can be used without loading it explicitly. + */ +public class InMemoryVariableStorage extends VariableStorage { + + private final VariablesMap variablesMap = new VariablesMap(); + + public InMemoryVariableStorage(SkriptAddon source, String type) { + super(source, type); + } + + @Override + protected boolean load(SectionNode n) { + return true; + } + + @Override + protected boolean requiresFile() { + return false; + } + + @Override + protected File getFile(String fileName) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable Object getVariable(String name) { + return variablesMap.getVariable(name); + } + + @Override + public void setVariable(String name, @Nullable Object value) { + variablesMap.setVariable(name, value); + } + + @Override + public long loadedVariables() { + return variablesMap.size(); + } + + @Override + public void close() { + } + +} diff --git a/src/main/java/org/skriptlang/skript/variables/storage/MySQLStorage.java b/src/main/java/org/skriptlang/skript/variables/storage/MySQLStorage.java new file mode 100644 index 00000000000..a9e7875b67a --- /dev/null +++ b/src/main/java/org/skriptlang/skript/variables/storage/MySQLStorage.java @@ -0,0 +1,70 @@ +package org.skriptlang.skript.variables.storage; + +import java.io.File; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.JdbcStorage; + +import com.zaxxer.hikari.HikariConfig; + +import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; + +/** + * MySQL storage for Skript variables. + */ +@SuppressWarnings("SqlSourceToSinkFlow") +public class MySQLStorage extends JdbcStorage { + + public MySQLStorage(SkriptAddon source, String type) { + super(source, type); + } + + @Override + protected @Nullable HikariConfig configuration(SectionNode sectionNode) { + String host = getValue(sectionNode, "host"); + Integer port = getValue(sectionNode, "port", Integer.class); + String database = getValue(sectionNode, "database"); + if (host == null || port == null || database == null) + return null; + + HikariConfig configuration = new HikariConfig(); + configuration.setJdbcUrl("jdbc:mysql://" + host + ":" + port + "/" + database); + configuration.setUsername(getValue(sectionNode, "user")); + configuration.setPassword(getValue(sectionNode, "password")); + + return configuration; + } + + @Override + protected boolean requiresFile() { + return false; + } + + @Override + protected File getFile(String fileName) { + throw new UnsupportedOperationException(); + } + + // language=MySQL + @Override + protected String createTableQuery() { + return "CREATE TABLE IF NOT EXISTS " + table + " (" + + "name VARCHAR(" + MAX_VARIABLE_NAME_LENGTH + ") PRIMARY KEY, " + + "type VARCHAR(" + MAX_CLASS_CODENAME_LENGTH + "), " + + "value BLOB(" + MAX_VALUE_SIZE + ")" + + ") CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"; + } + + @Override + protected PreparedStatement writeSingleQuery(Connection connection) throws SQLException { + return connection.prepareStatement( + "INSERT INTO " + table + " (name, type, value) VALUES (?, ?, ?) " + + "ON DUPLICATE KEY UPDATE type=VALUES(type), value=VALUES(value)" + ); + } + +} diff --git a/src/main/java/org/skriptlang/skript/variables/storage/SQLiteStorage.java b/src/main/java/org/skriptlang/skript/variables/storage/SQLiteStorage.java new file mode 100644 index 00000000000..4b548909633 --- /dev/null +++ b/src/main/java/org/skriptlang/skript/variables/storage/SQLiteStorage.java @@ -0,0 +1,43 @@ +package org.skriptlang.skript.variables.storage; + +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.JdbcStorage; +import com.zaxxer.hikari.HikariConfig; +import org.jetbrains.annotations.Nullable; +import org.skriptlang.skript.addon.SkriptAddon; + +import java.io.File; + +/** + * SQLite storage for Skript variables. + */ +public class SQLiteStorage extends JdbcStorage { + + public SQLiteStorage(SkriptAddon source, String type) { + super(source, type); + } + + @Override + protected @Nullable HikariConfig configuration(SectionNode sectionNode) { + if (file == null) + return null; + assert file.getName().endsWith(".db"); + + HikariConfig configuration = new HikariConfig(); + configuration.setJdbcUrl("jdbc:sqlite:" + (file == null ? ":memory:" : file.getAbsolutePath())); + return configuration; + } + + @Override + protected boolean requiresFile() { + return true; + } + + @Override + protected File getFile(String fileName) { + if (!fileName.endsWith(".db")) + fileName = fileName + ".db"; // required by SQLite + return new File(fileName); + } + +} diff --git a/src/main/resources/config.sk b/src/main/resources/config.sk index 8d2e766e7b2..b92d1db2e87 100644 --- a/src/main/resources/config.sk +++ b/src/main/resources/config.sk @@ -365,27 +365,34 @@ databases: database: skript table: variables21 - monitor changes: true monitor interval: 20 seconds - SQLite example: - # An SQLite database example. + #commit changes: 0.5 seconds + # If you want to change how frequently SQL changes are commited. + # If this is disabled, auto commit will be enabled. + # Pros to this would be on external databases such as MySQL where it gives other servers time to react to changes. + + H2 example: + # An H2 database example. type: disabled # change to line below to enable this database - # type: SQLite + # type: H2 - pattern: db_.* # this pattern will save all variables that start with 'db_' in this SQLite database. + pattern: db_.* # this pattern will save all variables that start with 'db_' in this H2 database. - file: ./plugins/Skript/variables.db - # SQLite databases must end in '.db' + file: ./plugins/Skript/variables + # H2 suffixes the database file with '.mv.db' #table: variables21 # Usually not required, if omitted defaults to variables21 (see above for more details) - backup interval: 0 # 0 = don't create backups - monitor changes: false - monitor interval: 20 seconds + # optional H2 settings if you want. + #user: skript + #password: password + #description: skript - backups to keep: -1 + # If H2 should run in ram memory only mode. + #memory: true + backup interval: 0 # 0 = don't create backups default: # The default "database" is a simple text file, with each variable on a separate line and the variable's name, type, and value separated by commas. diff --git a/src/main/resources/plugin.yml b/src/main/resources/plugin.yml index d5602bc642f..bea1ccc0e95 100644 --- a/src/main/resources/plugin.yml +++ b/src/main/resources/plugin.yml @@ -31,6 +31,10 @@ main: ch.njol.skript.Skript version: @version@ api-version: 1.19 +libraries: + - 'com.h2database:h2:@h2.version@' + - 'com.zaxxer:HikariCP:@hikaricp.version@' + commands: skript: description: Skript's main command. Type '/skript help' for more information. diff --git a/src/test/java/ch/njol/skript/variables/StorageAccessor.java b/src/test/java/ch/njol/skript/variables/StorageAccessor.java new file mode 100644 index 00000000000..2539ff58307 --- /dev/null +++ b/src/test/java/ch/njol/skript/variables/StorageAccessor.java @@ -0,0 +1,12 @@ +package ch.njol.skript.variables; + +/** + * Class used by Variable tests to access package exclusive methods. + */ +public class StorageAccessor { + + public static void clearVariableStorages() { + Variables.STORAGES.clear(); + } + +} diff --git a/src/test/java/org/skriptlang/skript/variables/storage/H2StorageTest.java b/src/test/java/org/skriptlang/skript/variables/storage/H2StorageTest.java new file mode 100644 index 00000000000..67c4a8a2a92 --- /dev/null +++ b/src/test/java/org/skriptlang/skript/variables/storage/H2StorageTest.java @@ -0,0 +1,53 @@ +package org.skriptlang.skript.variables.storage; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; + +import ch.njol.skript.Skript; +import ch.njol.skript.config.Config; +import ch.njol.skript.config.ConfigReader; +import ch.njol.skript.config.EntryNode; +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.StorageAccessor; + +public class H2StorageTest { + + private static final boolean ENABLED = Skript.classExists("com.zaxxer.hikari.HikariConfig"); + private final String testSection = + "h2:\n" + + "\tpattern: .*\n" + + "\tfile: ./plugins/Skript/variables\n" + + "\tbackup interval: 0"; + + private H2Storage database; + + @Before + public void setup() { + if (!ENABLED) + return; + Config config; + try { + config = new Config(new ByteArrayInputStream(testSection.getBytes(ConfigReader.UTF_8)), "h2-junit.sk", false, false, ":"); + } catch (IOException e) { + throw new RuntimeException(e); + } + StorageAccessor.clearVariableStorages(); + database = new H2Storage(Skript.instance(), "H2"); + SectionNode section = new SectionNode("h2", "", config.getMainNode(), 0); + section.add(new EntryNode("pattern", ".*", section)); + section.add(new EntryNode("file", "./plugins/Skript/variables", section)); + section.add(new EntryNode("backup interval", "0", section)); + assertTrue(database.loadConfig(section)); + } + + @Test + public void testStorage() { + // TODO + } + +} diff --git a/src/test/java/org/skriptlang/skript/variables/storage/SQLiteStorageTest.java b/src/test/java/org/skriptlang/skript/variables/storage/SQLiteStorageTest.java new file mode 100644 index 00000000000..07a5fc004cd --- /dev/null +++ b/src/test/java/org/skriptlang/skript/variables/storage/SQLiteStorageTest.java @@ -0,0 +1,54 @@ +package org.skriptlang.skript.variables.storage; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; +import ch.njol.skript.Skript; +import ch.njol.skript.config.Config; +import ch.njol.skript.config.ConfigReader; +import ch.njol.skript.config.EntryNode; +import ch.njol.skript.config.SectionNode; +import ch.njol.skript.variables.StorageAccessor; + +public class SQLiteStorageTest { + + private static final boolean ENABLED = Skript.classExists("com.zaxxer.hikari.HikariConfig"); + private final String testSection = + "sqlite:\n" + + "\tpattern: .*\n" + + "\tmonitor interval: 30 seconds\n" + + "\tfile: ./plugins/Skript/variables.db\n" + + "\tbackup interval: 0"; + + private SQLiteStorage database; + + @Before + public void setup() { + if (!ENABLED) + return; + Config config; + try { + config = new Config(new ByteArrayInputStream(testSection.getBytes(ConfigReader.UTF_8)), "sqlite-junit.sk", false, false, ":"); + } catch (IOException e) { + throw new RuntimeException(e); + } + StorageAccessor.clearVariableStorages(); + database = new SQLiteStorage(Skript.instance(), "H2"); + SectionNode section = new SectionNode("sqlite", "", config.getMainNode(), 0); + section.add(new EntryNode("pattern", ".*", section)); + section.add(new EntryNode("monitor interval", "30 seconds", section)); + section.add(new EntryNode("file", "./plugins/Skript/variables.db", section)); + section.add(new EntryNode("backup interval", "0", section)); + assertTrue(database.loadConfig(section)); + } + + @Test + public void testStorage() { + // TODO + } + +}