diff --git a/docs/groovy/conceptual/execution-context.md b/docs/groovy/conceptual/execution-context.md index 347d91ac2bb..ca537fa88a3 100644 --- a/docs/groovy/conceptual/execution-context.md +++ b/docs/groovy/conceptual/execution-context.md @@ -235,7 +235,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile(), ClassLoader.getSystemClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ``` @@ -248,7 +248,7 @@ This approach allows you to specify: - **Query scope**: A new, empty query scope via `newQueryScope`. - **Operation initializer**: Parallelization behavior of operations. - **Update graph**: A custom update graph (e.g., `PeriodicUpdateGraph` or `EventDrivenUpdateGraph`). -- **Query compiler**: A compiler instance with a specified working directory and class loader. +- **Query compiler**: A compiler instance with an optional directory to load classes from and classloader. For use cases requiring event-driven updates instead of periodic updates, you can substitute an `EventDrivenUpdateGraph`: @@ -262,7 +262,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(eventDrivenGraph) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile(), ClassLoader.getSystemClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ``` diff --git a/docs/python/conceptual/execution-context.md b/docs/python/conceptual/execution-context.md index eb7f69f6c4e..ef50996cda2 100644 --- a/docs/python/conceptual/execution-context.md +++ b/docs/python/conceptual/execution-context.md @@ -260,12 +260,7 @@ execution_context = ( .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler( - QueryCompilerImpl.create( - jpy.get_type("java.io.File")(temp_dir), - jpy.get_type("java.lang.ClassLoader").getSystemClassLoader(), - ) - ) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ) ``` @@ -279,7 +274,7 @@ This approach allows you to specify: - **Query scope**: A new, empty query scope via `newQueryScope`. - **Operation initializer**: Parallelization behavior of operations. - **Update graph**: A custom update graph (e.g., `PeriodicUpdateGraph` or `EventDrivenUpdateGraph`). -- **Query compiler**: A compiler instance with a specified working directory and class loader. +- **Query compiler**: A custom query compiler, which can be created with an optional class directory, plus optional classloader For use cases requiring event-driven updates instead of periodic updates, you can substitute an `EventDrivenUpdateGraph`: @@ -306,12 +301,7 @@ execution_context = ( .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(event_driven_graph) - .setQueryCompiler( - QueryCompilerImpl.create( - jpy.get_type("java.io.File")(temp_dir), - jpy.get_type("java.lang.ClassLoader").getSystemClassLoader(), - ) - ) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ) ``` diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 4041f91cf02..638946bb13e 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -10,6 +10,16 @@ import java.util.concurrent.ExecutionException; +/** + * Java compiler api, specifically geared towards query language expressions. Implementations should be constructed with + * a path on disk to existing bytecode to use in addition to {@code java.class.path}. Likewise, callers must ensure that + * the context classloader already contains those input classes - the new class will be loaded in a child classloader to + * ensure that the class can be GCd when neither it nor the compiler instance are reachable. + *

+ * Requests for compilation will never depend on other classes compiled by this or any other QueryCompiler, and are + * expected to be unique based on the provided class name. This enables the implementation to cache classes and only + * compile and load a class once per name. + */ public interface QueryCompiler { /** diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index fa39ad0211f..11af4585216 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -14,9 +14,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import java.io.IOException; import java.lang.reflect.Method; import java.time.Instant; import java.time.LocalDateTime; @@ -60,18 +58,14 @@ private static String generateClassBody(final String className) { @Rule public final EngineCleanup framework = new EngineCleanup(); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - private SafeCloseable executionContextClosable; @Before - public void setUp() throws IOException { + public void setUp() { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(QueryCompilerImpl.create( - folder.newFolder(), TestQueryCompiler.class.getClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create()) .build() .open(); } @@ -375,12 +369,15 @@ public void testBadCompile() { CompletionStageFuture.make(), }; - final QueryCompilerImpl badCompiler = QueryCompilerImpl.createForUnitTests(List.of("InvalidClassArgument")); + final QueryCompilerImpl badCompiler = QueryCompilerImpl.create(); + badCompiler.setClassNamesForAnnotationProcessing(List.of("InvalidClassArgument")); + UncheckedDeephavenException e = org.junit.Assert.assertThrows(UncheckedDeephavenException.class, () -> badCompiler.compile(requests, resolvers)); org.junit.Assert.assertEquals("Error Invoking Compiler, no source present in diagnostic:\n" + "Class names, 'InvalidClassArgument', are only accepted if annotation processing is explicitly requested", e.getMessage()); + } @Test diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 6cce3289372..68ad3a58899 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -3,13 +3,13 @@ // package io.deephaven.engine.context; +import com.google.common.hash.Hashing; import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; -import io.deephaven.configuration.DataDir; import io.deephaven.engine.context.util.SynchronizedJavaFileManager; import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; @@ -17,39 +17,69 @@ import io.deephaven.engine.table.impl.util.JobScheduler; import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.log.LogEntry; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; -import io.deephaven.util.ByteUtils; import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; - -import javax.tools.*; -import java.io.*; +import org.jetbrains.annotations.Nullable; + +import javax.tools.Diagnostic; +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardLocation; +import javax.tools.ToolProvider; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; -import java.net.MalformedURLException; import java.net.URI; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory and defines the resulting + * classes via {@link ClassLoader#defineClass}, avoiding filesystem writes for compilation output. + * + *

+ * The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory + * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the + * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Classes nor this + * compiler instance are reachable. + */ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { private static final Logger log = LoggerFactory.getLogger(QueryCompilerImpl.class); @@ -59,19 +89,9 @@ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); - private static final int MAX_CLASS_COLLISIONS = 128; private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; - private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; - private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds - private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; - private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long CODEGEN_TIMEOUT_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long CODEGEN_LOOP_DELAY_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); - private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); private static final String TRACE_PREFIX_PROPERTY = "QueryCompiler.tracePrefixes"; @@ -108,7 +128,6 @@ private static boolean shouldTrace(String className) { return TRACE_INCLUDE_PREFIXES.stream().anyMatch(className::startsWith); } - private static JavaCompiler compiler; private static final AtomicReference fileManagerCache = new AtomicReference<>(); @@ -147,57 +166,135 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; - public static QueryCompilerImpl create(File cacheDirectory, ClassLoader classLoader) { - return new QueryCompilerImpl(cacheDirectory, classLoader, true, null); + /** + * Creates a new QueryCompilerImpl. Uses the current thread's context classloader as the parent for per-batch + * classloaders (as required by {@link QueryCompiler}'s contract). + * + * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) + */ + public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir) { + return create(additionalClassPathDir, null); } - static QueryCompilerImpl createForUnitTests() { - return createForUnitTests(null); + /** + * Creates a new QueryCompilerImpl. Uses the current thread's context classloader as the parent for per-batch + * classloaders (as required by {@link QueryCompiler}'s contract). + * + * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) + */ + public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir, + @Nullable ClassLoader classLoader) { + return new QueryCompilerImpl(additionalClassPathDir, classLoader); } - static QueryCompilerImpl createForUnitTests(final List classNamesForAnnotationProcessing) { - final Path queryCompilerDir = DataDir.get() - .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); - return new QueryCompilerImpl(queryCompilerDir.toFile(), QueryCompilerImpl.class.getClassLoader(), false, - classNamesForAnnotationProcessing); + /** + * Creates a new compiler that has no extra directory to read from, suitable for unit tests or cases where the + * existing classpath is sufficient. + */ + public static QueryCompilerImpl create() { + return create(null, null); } - private final Map>> knownClasses = new HashMap<>(); + /** + * Cache from SHA-256 hash of class body to entry. While compilation is in-flight, the entry holds a future for + * coordination. Once complete, the entry transitions to a weak reference so the class (and its classloader and + * bytecode) can be GC'd when no longer in use. All transitions are atomic via per-entry synchronization. Stale + * entries are cleaned up via a {@link ReferenceQueue} drained at the start of each {@link #compile} call. + */ + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); - private final String[] dynamicPatterns = new String[] {DYNAMIC_CLASS_PREFIX, FORMULA_CLASS_PREFIX}; + /** Queue that receives cleared weak references, allowing us to remove stale cache entries. */ + private final ReferenceQueue> staleQueue = new ReferenceQueue<>(); - private final File classDestination; - private final Set additionalClassLocations; - private final WritableURLClassLoader ucl; // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. - private final List classNamesForAnnotationProcessing; + private List classNamesForAnnotationProcessing; - private QueryCompilerImpl( - @NotNull final File classDestination, - @NotNull final ClassLoader parentClassLoader, - boolean classDestinationIsAlsoClassSource, - final List classNamesForAnnotationProcessing) { - ensureJavaCompiler(); + /** + * A WeakReference that remembers its cache key, so we can remove the entry when the referent is GC'd. + */ + private static final class KeyedWeakReference extends WeakReference> { + final String key; - this.classDestination = classDestination; - ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + - classDestination.getAbsolutePath()); - additionalClassLocations = new LinkedHashSet<>(); + KeyedWeakReference(@NotNull Class referent, @NotNull String key, + @NotNull ReferenceQueue> queue) { + super(referent, queue); + this.key = key; + } + } - URL[] urls = new URL[1]; - try { - urls[0] = (classDestination.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); + /** + * A cache entry that transitions from in-flight (future-based coordination) to weakly-cached (reclaimable). + * Thread-safe: all reads/writes are synchronized on the entry itself. The entry self-transitions to weak-ref state + * when its future completes successfully, via a callback registered at construction time. + * + *

+ * Safety invariant: {@code CompletionStageFutureImpl} (extends {@code CompletableFuture}) retains its result + * strongly in an internal field even after listeners fire. This means any thread holding a reference to the future + * can still retrieve the class via {@code get()}, even after this entry has transitioned to weak-ref state. The + * weak reference only governs the cache's retention — not in-flight callers'. + *

+ */ + private static final class CacheEntry { + private CompletionStageFuture> future; + private WeakReference> classRef; + + CacheEntry(@NotNull CompletionStageFuture> future, + @NotNull String key, + @NotNull ReferenceQueue> queue) { + this.future = future; + // Self-transition: when compilation completes, swap to keyed weak ref + future.whenComplete((clazz, err) -> { + if (clazz != null) { + complete(clazz, key, queue); + } + // On failure, the entry stays in-flight with a failed future. + // The caller will remove it from the cache. + }); + } + + /** Transition to weak-ref state: releases the strong reference held by the completed future. */ + private synchronized void complete( + @NotNull Class clazz, @NotNull String key, @NotNull ReferenceQueue> queue) { + this.classRef = new KeyedWeakReference(clazz, key, queue); + this.future = null; + } + + /** + * Atomically resolve this entry's state. Returns one of: + *
    + *
  • A {@code Class} — cache hit, class still alive
    • + *
  • A {@code CompletionStageFuture>} — compilation in flight, wait on it
    • + *
  • {@code null} — entry is stale (class was GC'd), caller should retry
    • + *
+ */ + synchronized Optional>> resolve() { + if (classRef != null) { + final Class c = classRef.get(); + return Optional.ofNullable(c).map(CompletableFuture::completedFuture); + } + return Optional.of(future); } - this.ucl = new WritableURLClassLoader(urls, parentClassLoader); - log.trace().append("Class destination is ").append(classDestination.toString()).endl(); - if (classDestinationIsAlsoClassSource) { - addClassSource(classDestination); + /** True if completed but the class has been GC'd (stale). Used inside compute() lambda. */ + synchronized boolean isStale() { + return future == null && (classRef == null || classRef.get() == null); } + } - this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; + /** The context classloader provided at creation time, or null if none set */ + @Nullable + private final ClassLoader parentClassLoader; + + /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ + @Nullable + private final File additionalClassPathDir; + + private QueryCompilerImpl( + @Nullable final File additionalClassPathDir, + @Nullable final ClassLoader parentClassLoader) { + ensureJavaCompiler(); + this.additionalClassPathDir = additionalClassPathDir; + this.parentClassLoader = parentClassLoader; if (log.isTraceEnabled()) { log.trace().append("QueryCompiler Class Path: ").append(getClassPath()).append(File.pathSeparator) @@ -205,9 +302,29 @@ private QueryCompilerImpl( } } + /** Drains the stale queue and removes any cache entries whose weak references have been cleared by GC. */ + private void evictStaleEntries() { + KeyedWeakReference ref; + while ((ref = (KeyedWeakReference) staleQueue.poll()) != null) { + cache.remove(ref.key); + } + } + + /** + * Currently only intended for testing, this method allows specifying annotation processors to run during this + * compile. + * + * @param classNames the annotation processor classes to allow to run + */ + @TestUseOnly + public void setClassNamesForAnnotationProcessing(List classNames) { + classNamesForAnnotationProcessing = classNames; + } + @Override public LogOutput append(LogOutput logOutput) { - return logOutput.append("QueryCompiler{classDestination=").append(classDestination.getAbsolutePath()) + return logOutput.append("QueryCompilerImpl{additionalClassPathDir=") + .append(additionalClassPathDir == null ? "null" : additionalClassPathDir.getAbsolutePath()) .append("}"); } @@ -228,78 +345,6 @@ public static boolean setLogEnabled(boolean logEnabled) { return original; } - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data) - throws IOException { - writeClass(destinationDirectory, className, data, null); - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data, - final String message) throws IOException { - final File destinationFile = new File(destinationDirectory, - className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - - final boolean shouldTrace = shouldTrace(className); - - if (destinationFile.exists()) { - if (shouldTrace) { - log.trace().append("Destination class file already exists ").append(destinationFile.toString()).endl(); - } - - final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); - if (Arrays.equals(existingBytes, data)) { - if (message == null) { - log.info().append("Ignoring pushed class ").append(className) - .append(" because it already exists in this context!").endl(); - } else { - log.info().append("Ignoring pushed class ").append(className).append(message) - .append(" because it already exists in this context!").endl(); - } - return; - } else { - if (message == null) { - log.info().append("Pushed class ").append(className) - .append(" already exists in this context, but has changed!").endl(); - } else { - log.info().append("Pushed class ").append(className).append(message) - .append(" already exists in this context, but has changed!").endl(); - } - if (!destinationFile.delete()) { - throw new IOException("Could not delete existing class file: " + destinationFile); - } - } - } else if (shouldTrace) { - log.trace().append("Destination class file does not already exist ").append(destinationFile.toString()) - .endl(); - } - - final File parentDir = destinationFile.getParentFile(); - ensureDirectories(parentDir, - () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); - if (!destinationFile.createNewFile()) { - throw new UncheckedDeephavenException( - "Unable to create destination file " + destinationFile.getAbsolutePath()); - } - final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); - byteOutStream.write(data, 0, data.length); - final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); - byteOutStream.writeTo(fileOutStream); - fileOutStream.close(); - - if (shouldTrace) { - log.trace().append("Wrote destination class file ").append(destinationFile.toString()).endl(); - } - } - @Override public void compile( @NotNull final QueryCompilerRequest[] requests, @@ -311,6 +356,9 @@ public void compile( throw new IllegalArgumentException("Requests and resolvers must be the same length"); } + // Opportunistically clean up stale cache entries whose classes have been GC'd + evictStaleEntries(); + final boolean shouldTrace = log.isTraceEnabled() && Arrays.stream(requests).map(QueryCompilerRequest::packageNameRoot) .anyMatch(QueryCompilerImpl::shouldTrace); @@ -327,58 +375,80 @@ public void compile( }, requests).endl(); } - // noinspection unchecked - final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; - + // For each request: cache hit, in-flight wait, or needs compilation final List newRequests = new ArrayList<>(); final List>> newResolvers = new ArrayList<>(); + final List newHashes = new ArrayList<>(); + final List>> allFutures = new ArrayList<>(requests.length); - synchronized (this) { - for (int ii = 0; ii < requests.length; ++ii) { - final QueryCompilerRequest request = requests[ii]; - final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + for (int ii = 0; ii < requests.length; ++ii) { + // Compute content hash for deduplication + QueryCompilerRequest req = requests[ii]; + String hash = Hashing.sha256().hashString(req.classBody(), StandardCharsets.UTF_8).toString(); + + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + // Resolve cache state, retrying if we hit a stale entry race + Optional>> resolved; + while (true) { + final CacheEntry entry = cache.compute(hash, (key, existing) -> { + if (existing != null && !existing.isStale()) { + return existing; + } - CompletionStageFuture> future = - knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); - if (future == null) { - newRequests.add(request); + // No/stale existing entry, we will do the compilation ourselves newResolvers.add(resolver); - future = resolver.getFuture(); - } else if (shouldTrace) { - log.trace().append("Found existing future in knownClasses for ").append(request.className()) - .append(" (done=").append(future.isDone()).append(")").endl(); + newRequests.add(req); + newHashes.add(hash); + return new CacheEntry(resolver.getFuture(), key, staleQueue); + }); + + resolved = entry.resolve(); + if (resolved.isPresent()) { + // Got a future, either to our work, or someone else's + allFutures.add(resolved.get()); + break; } - allFutures[ii] = future; + // Stale race: entry transitioned between compute() and resolve(). Remove and retry. + cache.remove(hash, entry); } } + // Compile new requests if (!newResolvers.isEmpty()) { - // It's my job to fulfill these futures. try { - compileHelper(newRequests, newResolvers); + compileHelper(newRequests, newResolvers, newHashes); } catch (RuntimeException e) { - // These failures are not applicable to a single request, so we can't just complete the future and - // leave the failure in the cache. - synchronized (this) { - for (int ii = 0; ii < newRequests.size(); ++ii) { - if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newRequests.get(ii).classBody()); - } + for (int ii = 0; ii < newHashes.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + cache.remove(newHashes.get(ii)); } } throw e; } + + // Remove cache entries for failed or incomplete compilations + // (successful ones self-transitioned to weak refs via whenComplete) + for (int ii = 0; ii < newHashes.size(); ++ii) { + try { + newResolvers.get(ii).getFuture().get(); // throws if completed exceptionally + } catch (Exception ex) { + cache.remove(newHashes.get(ii)); + } + } } + // Wait on in-flight futures from other compilations for (int ii = 0; ii < requests.length; ++ii) { + final Future> future = allFutures.get(ii); try { - resolvers[ii].complete(allFutures[ii].get()); + resolvers[ii].complete(future.get()); } catch (ExecutionException err) { resolvers[ii].completeExceptionally(err.getCause()); } catch (InterruptedException err) { // This can only occur if we are interrupted while waiting for the future to complete from another // compilation request. - Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures.get(ii), "allFutures.get(ii)"); resolvers[ii].completeExceptionally(err); } catch (Throwable err) { resolvers[ii].completeExceptionally(err); @@ -386,562 +456,365 @@ public void compile( } } - private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { - // File.mkdirs() checks for existence on entry, in which case it returns false. - // It may also return false on a failure to create. - // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could - // lose the race - // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() - // would return true) - if (!file.mkdirs() && !file.isDirectory()) { - throw new UncheckedDeephavenException(runtimeErrMsg.get()); - } - } - - private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { - return new URLClassLoader(ucl.getURLs(), ucl) { - // Once we find a class that is missing, we should not attempt to load it again, - // otherwise we can end up with a StackOverflow Exception - final HashSet missingClasses = new HashSet<>(); - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - final boolean shouldTrace = shouldTrace(name); - - // If we have a parameter that uses this class, return it - final Class paramClass = parameterClasses.get(name); - if (paramClass != null) { - if (shouldTrace) { - log.trace().append("findClass(").append(name).append("): matched parameter class").endl(); - } - return paramClass; - } + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers, + @NotNull final List hashes) { + // Assign unique FQ class names for each request using pre-computed hashes + final String[] fqClassNames = new String[requests.size()]; + final String[] packageNames = new String[requests.size()]; - // Unless we are looking for a formula or Groovy class, we should use the default behavior - if (!isFormulaClass(name)) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): not a formula class, delegating to super.findClass").endl(); - } - return super.findClass(name); - } + for (int ii = 0; ii < requests.size(); ++ii) { + final QueryCompilerRequest request = requests.get(ii); + final String packageNameSuffix = "c_" + hashes.get(ii) + "v" + JAVA_CLASS_VERSION; + packageNames[ii] = request.getPackageName(packageNameSuffix); + fqClassNames[ii] = packageNames[ii] + "." + request.className(); + } - // if it is a groovy class, always try to use the instance in the shell - if (name.startsWith(DYNAMIC_CLASS_PREFIX)) { - try { - final Class dynamicResult = ucl.getParent().loadClass(name); - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): loaded dynamic class from parent of WritableURLClassLoader").endl(); - } - return dynamicResult; - } catch (final ClassNotFoundException ignored) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): dynamic class not found in parent of WritableURLClassLoader," - + " falling through") - .endl(); - } - // we'll try to load it otherwise - } - } + // Build compilation attempts + final List attempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + attempts.add(new CompilationRequestAttempt( + requests.get(ii), packageNames[ii], fqClassNames[ii], resolvers.get(ii))); + } - // We've already not found this class, so we should not try to search again - if (missingClasses.contains(name)) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): previously cached as missing, delegating to super.findClass").endl(); - } - return super.findClass(name); - } + // Compile and define + compileAndDefine(attempts); - final byte[] bytes; - try { - bytes = loadClassData(name); - } catch (IOException ioe) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): loadClassData failed (").append(ioe.getMessage()) - .append("), caching as missing and delegating to super.loadClass").endl(); - } - missingClasses.add(name); - return super.loadClass(name); - } - if (shouldTrace) { - log.trace().append("findClass(").append(name).append("): defining class from ") - .append(bytes.length).append(" bytes").endl(); - } - return defineClass(name, bytes, 0, bytes.length); + // Validate _CLASS_BODY_ field on successfully defined classes + for (int ii = 0; ii < requests.size(); ++ii) { + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + continue; // already completed (success or failure) } + // This shouldn't happen - compileAndDefine should have resolved everything + resolver.completeExceptionally(new IllegalStateException( + "Class was not resolved after compilation: " + fqClassNames[ii])); + } + } - @SuppressWarnings("BooleanMethodIsAlwaysInverted") - private boolean isFormulaClass(String name) { - return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); - } + private void compileAndDefine(@NotNull final List requests) { + final ExecutionContext executionContext = ExecutionContext.getContext(); + final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); - @Override - public Class loadClass(String name) throws ClassNotFoundException { - if (!isFormulaClass(name)) { - return super.loadClass(name); - } - return findClass(name); - } + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - private byte[] loadClassData(String name) throws IOException { - final boolean shouldTrace = shouldTrace(name); + final int numTasks; + final JobScheduler jobScheduler; - final File destFile = new File(classDestination, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (destFile.exists()) { - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): found at ") - .append(destFile.toString()).endl(); - } - return Files.readAllBytes(destFile.toPath()); - } - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): not present at ") - .append(destFile.toString()).endl(); - } + final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); + if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { + numTasks = 1; + jobScheduler = new ImmediateJobScheduler(); + } else { + numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + jobScheduler = new OperationInitializerJobScheduler(); + } - for (File location : additionalClassLocations) { - final File checkFile = new File(location, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (checkFile.exists()) { - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): found at ") - .append(checkFile.toString()).endl(); - } - return Files.readAllBytes(checkFile.toPath()); - } - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): not present at ") - .append(checkFile.toString()).endl(); - } - } + log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") + .append(numTasks).append(" tasks, ").append(requestsPerTask).endl(); - if (shouldTrace) { - log.trace().append("loadClassData(").append(name) - .append("): not found in any class location").endl(); - } - throw new FileNotFoundException(name); + final JavaFileManager fileManager = acquireFileManager(); + final AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + final Runnable cleanup = () -> { + try { + releaseFileManager(fileManager); + } catch (Exception e) { + // ignore errors here + } finally { + latch.countDown(); } }; - } - - private static class WritableURLClassLoader extends URLClassLoader { - private WritableURLClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - } - @Override - protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - final boolean shouldTrace = shouldTrace(name); - final int loaderId = System.identityHashCode(this); - - Class clazz = findLoadedClass(name); - if (clazz != null) { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") returned from findLoadedClass").endl(); - } - return clazz; + final Consumer onError = err -> { + if (err instanceof RuntimeException) { + exception.set((RuntimeException) err); + } else { + exception.set(new UncheckedDeephavenException("Error during compilation", err)); } + cleanup.run(); + }; - try { - clazz = findClass(name); - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") returned from findClass").endl(); - } - } catch (ClassNotFoundException e) { - if (getParent() != null) { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") delegating to parent").endl(); - } - clazz = getParent().loadClass(name); - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") parent returned class").endl(); - } - } else { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name) - .append(") not found by findClass and no parent loader; returning null").endl(); - } - } - } + jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, + 0, numTasks, (context, jobId, nestedErrorConsumer) -> { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + doCompileAndDefine(fileManager, requests, startInclusive, endExclusive); + }, + () -> { + }, + cleanup, + onError); - if (resolve) { - resolveClass(clazz); + try { + latch.await(); + final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); + if (perfEntry != null) { + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); } - return clazz; - } - - @Override - public synchronized void addURL(URL url) { - super.addURL(url); + final RuntimeException err = exception.get(); + if (err != null) { + throw err; + } + } catch (final InterruptedException e) { + throw new CancellationException("interrupted while compiling"); } } - private void addClassSource(File classSourceDirectory) { - synchronized (additionalClassLocations) { - if (additionalClassLocations.contains(classSourceDirectory)) { - return; - } - additionalClassLocations.add(classSourceDirectory); + private void doCompileAndDefine( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + final boolean wantRetry = doCompileAndDefineSingleRound( + fileManager, requests, startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; } - try { - ucl.addURL(classSourceDirectory.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); + // Retry non-failing requests from the first pass + final List ignored = new ArrayList<>(); + if (doCompileAndDefineSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { + throw new IllegalStateException("Unexpected failure during second pass of compilation"); } } - private File getClassDestination() { - return classDestination; - } + private boolean doCompileAndDefineSingleRound( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive, + @NotNull final List toRetry) { - private String getClassPath() { - StringBuilder sb = new StringBuilder(); - sb.append(classDestination.getAbsolutePath()); - synchronized (additionalClassLocations) { - for (File classLoc : additionalClassLocations) { - sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); - } - } - return sb.toString(); - } + // Create an in-memory file manager that captures compiled output + final InMemoryOutputFileManager outputFm = new InMemoryOutputFileManager(fileManager); + final StringWriter compilerOutput = new StringWriter(); - private static class CompilationState { - int nextProbeIndex; - boolean complete; - String packageName; - String fqClassName; - } + final String classPathAsString = getClassPath(); + final List compilerOptions = Arrays.asList( + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); - private void compileHelper( - @NotNull final List requests, - @NotNull final List>> resolvers) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } + final MutableInt numFailures = new MutableInt(0); + final List globalFailures = new ArrayList<>(); + compiler.getTask(compilerOutput, + outputFm, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } - final String[] basicHashText = new String[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( - requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + + if (source == null) { + // If we have no source, then mark every request as a failure. + final UncheckedDeephavenException err = new UncheckedDeephavenException( + "Error Invoking Compiler, no source present in diagnostic:\n" + + diagnostic.getMessage(Locale.getDefault())); + globalFailures.add(err); + return; + } + + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + classNamesForAnnotationProcessing, + requests.subList(startInclusive, endExclusive).stream() + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final String compilerOutputText = compilerOutput.toString(); + if (!compilerOutputText.isEmpty()) { + log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); } - int numComplete = 0; - final CompilationState[] states = new CompilationState[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - states[ii] = new CompilationState(); + if (!globalFailures.isEmpty()) { + final RuntimeException e0 = globalFailures.get(0); + for (int ii = 1; ii < globalFailures.size(); ++ii) { + e0.addSuppressed(globalFailures.get(ii)); + } + throw e0; } - /* - * @formatter:off - * 1. try to resolve without compiling; retain next hash to try - * 2. compile all remaining with a single compilation task - * 3. goto step 1 if any are unresolved - * @formatter:on - */ + final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; - while (numComplete < requests.size()) { - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { + // Define compiled classes into a per-batch classloader + final Map compiledClasses = outputFm.getCompiledClasses(); + log.trace().append("compilation produced ").append(compiledClasses.size()) + .append(" classes, numFailures=").append(numFailures.get()) + .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") + .endl(); + if (!compiledClasses.isEmpty()) { + // Use the current thread's context classloader as parent, unless we were explicitly provided a different + // one + final ClassLoader batchParent = + parentClassLoader != null ? parentClassLoader : Thread.currentThread().getContextClassLoader();; + final BatchClassLoader batchCl = new BatchClassLoader(batchParent, compiledClasses); + + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (request.resolver.getFuture().isDone()) { + // already failed continue; } - next_probe: while (true) { - final int pi = state.nextProbeIndex++; - final String packageNameSuffix = "c_" + basicHashText[ii] - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - - final QueryCompilerRequest request = requests.get(ii); - if (pi >= MAX_CLASS_COLLISIONS) { - Exception err = new IllegalStateException("Found too many collisions for package name root " - + request.packageNameRoot() + ", class name=" + request.className() + ", class body " - + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); - resolvers.get(ii).completeExceptionally(err); - state.complete = true; - ++numComplete; - break; + if (!compiledClasses.containsKey(request.fqClassName)) { + if (wantRetry) { + toRetry.add(request); } + continue; + } - state.packageName = request.getPackageName(packageNameSuffix); - state.fqClassName = state.packageName + "." + request.className(); - - for (int jj = 0; jj < ii; ++jj) { - if (states[jj].fqClassName.equals(state.fqClassName)) { - // collision within batch - continue next_probe; - } - } + // Load the outer class for this request + final Class clazz; + try { + clazz = batchCl.loadClass(request.fqClassName); + } catch (ClassNotFoundException e) { + request.resolver.completeExceptionally(new UncheckedDeephavenException( + "Failed to load compiled class: " + request.fqClassName, e)); + continue; + } - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - if (result == null) { - break; // we'll try to compile it - } - - final boolean shouldTrace = shouldTrace(state.fqClassName); - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), - result)) { - if (shouldTrace) { - log.trace().append("Successfully found existing class for ").append(state.fqClassName) - .endl(); - } - state.complete = true; - ++numComplete; - break; - } else if (shouldTrace) { - log.trace().append("Existing class did not match for ").append(state.fqClassName).endl(); - } + // Validate the identifying field + final String identifyingFieldValue = loadIdentifyingField(clazz); + if (!request.request.classBody().equals(identifyingFieldValue)) { + request.resolver.completeExceptionally(new IllegalStateException( + "Compiled class body validation failed for " + request.fqClassName)); + continue; } - } - if (numComplete == requests.size()) { - return; - } + // Notify caller with codeLog if requested + request.request.codeLog().ifPresent( + sb -> sb.append(makeFinalCode( + request.request.className(), request.request.classBody(), request.packageName))); - // Couldn't resolve at least one of the requests, so try a round of compilation. - final List compilationRequestAttempts = new ArrayList<>(); - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (!state.complete) { - final QueryCompilerRequest request = requests.get(ii); - compilationRequestAttempts.add(new CompilationRequestAttempt( - request, - state.packageName, - state.fqClassName, - resolvers.get(ii))); + // Complete the future + log.trace().append("Resolving ").append(request.fqClassName).endl(); + request.resolver.complete(clazz); + } + } else { + // No output at all - if there are non-failed requests, they need retry + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (!request.resolver.getFuture().isDone() && wantRetry) { + toRetry.add(request); } } + } - maybeCreateClasses(compilationRequestAttempts); + return wantRetry && !toRetry.isEmpty(); + } - // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it - // ... then give the filesystem some time. All requests should use the same deadline. - final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { - continue; - } + // --- Classpath construction --- - final QueryCompilerRequest request = requests.get(ii); - final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); - if (resolver.getFuture().isDone()) { - state.complete = true; - ++numComplete; - continue; - } + private String getClassPath() { + final StringBuilder sb = new StringBuilder(getJavaClassPath()); + if (additionalClassPathDir != null) { + sb.append(File.pathSeparator).append(additionalClassPathDir.getAbsolutePath()); + } + return sb.toString(); + } - // This request may have: - // A. succeeded - // B. Lost a race to another process on the same file system which is compiling the identical formula - // C. Lost a race to another process on the same file system compiling a different formula that collides - - final boolean shouldTrace = shouldTrace(state.fqClassName); - Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - if (clazz == null) { - final long waitStart = System.currentTimeMillis(); - if (shouldTrace) { - log.trace().append("Class not loadable immediately after compile;" - + " entering retry loop for ").append(state.fqClassName).endl(); - } - try { - while (clazz == null && System.currentTimeMillis() < deadline) { - // noinspection BusyWait - Thread.sleep(CODEGEN_LOOP_DELAY_MS); - clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - } - } catch (final InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); - } - if (shouldTrace) { - final long waited = System.currentTimeMillis() - waitStart; - if (clazz != null) { - log.trace().append("Class loaded after waiting ").append(waited) - .append(" ms: ").append(state.fqClassName).endl(); - } else { - log.trace().append("Retry loop timed out after ").append(waited) - .append(" ms: ").append(state.fqClassName).endl(); - } - } - } + // --- BatchClassLoader --- - // However, regardless of A-C, there will be *some* class being found - if (clazz == null) { - throw new IllegalStateException("Unable to load class after delay of " + CODEGEN_TIMEOUT_MS - + ". state index=" + ii + ", fqClassName=" + state.fqClassName + ", parameterClasses" - + request.parameterClasses() + ", destination=" + getClassDestination().getAbsolutePath()); - } + /** + * A classloader that defines classes from a map of name→bytes. All classes from a single compilation batch are + * loaded together. The parent classloader handles all other class resolution. + */ + private static class BatchClassLoader extends ClassLoader { + private final Map classBytes; - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { - if (shouldTrace) { - log.trace().append("Loaded matching compiled class ").append(state.fqClassName).endl(); - } - state.complete = true; - ++numComplete; - } else if (shouldTrace) { - log.trace().append("Existing class did not match for ").append(state.fqClassName).endl(); - } - } + BatchClassLoader(@NotNull ClassLoader parent, @NotNull Map classBytes) { + super(parent); + this.classBytes = classBytes; } - } - private boolean completeIfResultMatchesQueryCompilerRequest( - final String packageName, - final QueryCompilerRequest request, - final CompletionStageFuture.Resolver> resolver, - final Class result) { - final String identifyingFieldValue = loadIdentifyingField(result); - if (!request.classBody().equals(identifyingFieldValue)) { - final LogEntry logOutput = log.trace().append("Hash collision for ").append(result.getName()) - .append(": loaded identifying field length="); - if (identifyingFieldValue == null) { - logOutput.append("null"); - } else { - logOutput.append(identifyingFieldValue.length()); + @Override + protected Class findClass(String name) throws ClassNotFoundException { + final byte[] bytes = classBytes.get(name); + if (bytes != null) { + return defineClass(name, bytes, 0, bytes.length); } - logOutput.append(", expected class body length=").append(request.classBody().length()).endl(); - return false; + throw new ClassNotFoundException(name); } + } - // If the caller wants a textual copy of the code we either made, or just found in the cache. - request.codeLog() - .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - resolver.complete(result); - - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resolver.getFuture()); - } + // --- InMemoryOutputFileManager --- - return true; - } + /** + * A forwarding JavaFileManager that intercepts class output, storing compiled bytecode in memory rather than + * writing to the filesystem. All other operations are delegated. + */ + private static class InMemoryOutputFileManager extends ForwardingJavaFileManager { + private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); - private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { - final boolean shouldTrace = shouldTrace(fqClassName); - try { - if (shouldTrace) { - log.trace().append("Attempting to load ").append(fqClassName) - .append(" with parameter classes ") - .append(LogOutput.STRING_COLLECTION_FORMATTER, parameterClasses.keySet()).endl(); - } - return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); - } catch (ClassNotFoundException cnfe) { - if (shouldTrace) { - log.trace().append("Class not found for ").append(fqClassName).endl(); - } - return null; + InMemoryOutputFileManager(JavaFileManager delegate) { + super(delegate); } - } - private static String loadIdentifyingField(Class c) { - try { - final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); - return (String) field.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Malformed class in cache", e); + Map getCompiledClasses() { + final Map result = new HashMap<>(); + for (Map.Entry entry : outputClasses.entrySet()) { + result.put(entry.getKey(), entry.getValue().getBytes()); + } + return result; } - } - private static String makeFinalCode(String className, String classBody, String packageName) { - if (classBody.contains("$CLASSNAME$")) { - throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" - + " the final class name affects the compiled byte code and therefore cannot be dynamically " - + "replaced."); + @Override + public JavaFileObject getJavaFileForOutput( + Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { + if (location != StandardLocation.CLASS_OUTPUT || kind != JavaFileObject.Kind.CLASS) { + throw new IllegalArgumentException("In-memory output file manager only supports writing bytecode"); + } + final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); + InMemoryClassFileObject existing = outputClasses.put(className, fileObject); + if (existing != null) { + throw new IllegalArgumentException("Attempted to write duplicate class name " + className); + } + return fileObject; } - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; - } - - /** - * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping - * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java - * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. - */ - public static String createEscapedJoinedString(final String originalString) { - return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + @Override + public boolean hasLocation(Location location) { + if (location == StandardLocation.CLASS_OUTPUT) { + return true; + } + return super.hasLocation(location); + } } - public static String createEscapedJoinedString(final String originalString, int maxStringLength) { - final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); + // --- InMemoryClassFileObject --- - // Turn each split into a Java source string by escaping it and surrounding it with " - for (int ii = 0; ii < splits.length; ++ii) { - final String escaped = StringEscapeUtils.escapeJava(splits[ii]); - splits[ii] = "\"" + escaped + "\""; + private static class InMemoryClassFileObject extends SimpleJavaFileObject { + private final String className; + private ByteArrayOutputStream outputStream; + InMemoryClassFileObject(String className) { + super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); + this.className = className; } - assert splits.length > 0; - if (splits.length == 1) { - return splits[0]; + + @Override + public OutputStream openOutputStream() { + outputStream = new ByteArrayOutputStream(); + return outputStream; } - final String formattedInnards = String.join(",\n", splits); - return "String.join(\"\", " + formattedInnards + ")"; - } - private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { - final List splits = new ArrayList<>(); - // exclusive end position of the previous substring. - int previousEnd = 0; - // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. - int currentByteCount = 0; - for (int ii = 0; ii < originalString.length(); ++ii) { - final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); - if (currentByteCount + bytesConsumed > maxBytes) { - // This character won't fit in this string, so we flush the buffer. - splits.add(originalString.substring(previousEnd, ii)); - previousEnd = ii; - currentByteCount = 0; + byte[] getBytes() { + if (outputStream == null) { + throw new IllegalStateException("No bytes available for " + className); } - currentByteCount += bytesConsumed; + return outputStream.toByteArray(); } - // At the end of the loop, either - // 1. there are one or more characters that still need to be added to splits - // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits - splits.add(originalString.substring(previousEnd)); - return splits.toArray(String[]::new); } - private static int calcBytesConsumed(final char ch) { - if (ch == 0) { - return 2; - } - if (ch <= 0x7f) { - return 1; - } - if (ch <= 0x7ff) { - return 2; - } - return 3; - } + // --- Source representation --- private static class JavaSourceFromString extends SimpleJavaFileObject { final String description; @@ -959,6 +832,7 @@ private static class JavaSourceFromString extends SimpleJavaFileObject { this.resolver = resolver; } + @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) { return code; } @@ -969,7 +843,6 @@ private static class CompilationRequestAttempt { final String fqClassName; final String finalCode; final String packageName; - final String[] splitPackageName; final QueryCompilerRequest request; final CompletionStageFuture.Resolver> resolver; @@ -988,273 +861,88 @@ private CompilationRequestAttempt( if (logEnabled) { log.info().append("Generating code ").append(finalCode).endl(); - } else if (shouldTrace(fqClassName)) { - log.trace().append("Generating code for ").append(fqClassName).endl(); - } - - splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - final Exception err = new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); - resolver.completeExceptionally(err); - } - } - - public void ensureDirectories(@NotNull final String rootPath) { - if (splitPackageName.length == 0) { - // we've already failed - return; } - - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); - final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - QueryCompilerImpl.ensureDirectories(rpf, - () -> "Couldn't create package directories: " + rootPathWithPackage); } - public JavaSourceFromString makeSource() { + JavaSourceFromString makeSource() { return new JavaSourceFromString(description, fqClassName, finalCode, resolver); } } - private void maybeCreateClasses( - @NotNull final List requests) { - // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package - // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. - // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 - // This temp directory will be where the compiler drops files into, e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class - // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to - // /tmp/workspace/cache/classes/io/deephaven/test - // Note: for this atomic move to work, this temp directory must be on the same file system as the destination - // directory. - final String rootPathAsString; - final String tempDirAsString; - try { - rootPathAsString = getClassDestination().getAbsolutePath(); - for (final CompilationRequestAttempt request : requests) { - request.ensureDirectories(rootPathAsString); - } - - final Path tempPath = - Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); - tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - Exception err = new UncheckedIOException(ioe); - for (final CompilationRequestAttempt request : requests) { - request.resolver.completeExceptionally(err); - } - return; - } - - final ExecutionContext executionContext = ExecutionContext.getContext(); - final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); - - final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - - final int numTasks; - final JobScheduler jobScheduler; - - final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); - if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { - numTasks = 1; - jobScheduler = new ImmediateJobScheduler(); - } else { - numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; - jobScheduler = new OperationInitializerJobScheduler(); - } - - log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") - .append(numTasks).append(" tasks, ").append(requestsPerTask) - .append(" requests per task, tempDir=").append(tempDirAsString).endl(); - - final JavaFileManager fileManager = acquireFileManager(); - final AtomicReference exception = new AtomicReference<>(); - final CountDownLatch latch = new CountDownLatch(1); - final Runnable cleanup = () -> { - try { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here - } - try { - releaseFileManager(fileManager); - } catch (Exception e) { - // ignore errors here - } - } finally { - latch.countDown(); - } - }; - - final Consumer onError = err -> { - if (err instanceof RuntimeException) { - exception.set((RuntimeException) err); - } else { - exception.set(new UncheckedDeephavenException("Error during compilation", err)); - } - cleanup.run(); - }; - - jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, - 0, numTasks, (context, jobId, nestedErrorConsumer) -> { - final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - doCreateClasses( - fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); - }, - () -> { - }, - cleanup, - onError); + // --- Utilities --- + private static String loadIdentifyingField(Class c) { try { - latch.await(); - final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); - if (perfEntry != null) { - QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); - } - final RuntimeException err = exception.get(); - if (err != null) { - throw err; - } - } catch (final InterruptedException e) { - throw new CancellationException("interrupted while compiling"); + final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); + return (String) field.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Malformed class in cache", e); } } - private void doCreateClasses( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive) { - final List toRetry = new ArrayList<>(); - // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The - // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to - // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = doCreateClassesSingleRound(fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive, toRetry); - if (!wantRetry) { - return; + private static String makeFinalCode(String className, String classBody, String packageName) { + if (classBody.contains("$CLASSNAME$")) { + throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" + + " the final class name affects the compiled byte code and therefore cannot be dynamically " + + "replaced."); } - final List ignored = new ArrayList<>(); - if (doCreateClassesSingleRound(fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), - ignored)) { - // We only retried compilation units that did not fail on the first pass, so we should not have any failures - // on the second pass. - throw new IllegalStateException("Unexpected failure during second pass of compilation"); - } + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; } - private boolean doCreateClassesSingleRound( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive, - List toRetry) { - final StringWriter compilerOutput = new StringWriter(); - - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList( - "-d", tempDirAsString, - "-cp", classPathAsString, - // this option allows the compiler to attempt to process all source files even if some of them fail - "--should-stop=ifError=GENERATE"); - - final MutableInt numFailures = new MutableInt(0); - final List globalFailures = new ArrayList<>(); - compiler.getTask(compilerOutput, - fileManager, - diagnostic -> { - if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { - return; - } - - final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); - - if (source == null) { - // If we have no source, then mark every request as a failure. - final UncheckedDeephavenException err = new UncheckedDeephavenException( - "Error Invoking Compiler, no source present in diagnostic:\n" - + diagnostic.getMessage(Locale.getDefault())); - globalFailures.add(err); - return; - } + /** + * Transform a string into the corresponding Java source code that compiles into that string. + */ + public static String createEscapedJoinedString(final String originalString) { + return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + } - final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " - + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); - if (source.resolver.completeExceptionally(err)) { - // only count the first failure for each source - numFailures.increment(); - } - }, - compilerOptions, - classNamesForAnnotationProcessing, - requests.subList(startInclusive, endExclusive).stream() - .map(CompilationRequestAttempt::makeSource) - .collect(Collectors.toList())) - .call(); + public static String createEscapedJoinedString(final String originalString, int maxStringLength) { + final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - final String compilerOutputText = compilerOutput.toString(); - if (!compilerOutputText.isEmpty()) { - log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); + for (int ii = 0; ii < splits.length; ++ii) { + final String escaped = StringEscapeUtils.escapeJava(splits[ii]); + splits[ii] = "\"" + escaped + "\""; } - - if (!globalFailures.isEmpty()) { - final RuntimeException e0 = globalFailures.get(0); - for (int ii = 1; ii < globalFailures.size(); ++ii) { - e0.addSuppressed(globalFailures.get(ii)); - } - throw e0; + assert splits.length > 0; + if (splits.length == 1) { + return splits[0]; } + final String formattedInnards = String.join(",\n", splits); + return "String.join(\"\", " + formattedInnards + ")"; + } - final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; - - // The above has compiled into e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various - // class files} - // We want to atomically move it to e.g. - // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - requests.subList(startInclusive, endExclusive).forEach(request -> { - final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); - final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - if (shouldTrace(request.fqClassName)) { - log.trace().append("Successfully moved ").append(srcDir.toString()).append(" to ") - .append(destDir.toString()).endl(); - } - } catch (IOException ioe) { - // The name "isDone" might be misleading here. We haven't called "complete" on the successful - // futures yet, so the only way they would be "done" at this point is if they completed - // exceptionally. - final boolean hasException = request.resolver.getFuture().isDone(); - - if (wantRetry && !Files.exists(srcDir) && !hasException) { - // The move failed, the source directory does not exist, and this compilation unit actually - // succeeded. However, it was not written because some other compilation unit failed. Let's schedule - // this work to try again. - toRetry.add(request); - return; - } - - if (!Files.exists(destDir) && !hasException) { - // Propagate an error here only if the destination does not exist; ignoring issues related to - // collisions with another process. - request.resolver.completeExceptionally(new UncheckedIOException( - "Move failed for some reason other than destination already existing", ioe)); - } + private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { + final List splits = new ArrayList<>(); + int previousEnd = 0; + int currentByteCount = 0; + for (int ii = 0; ii < originalString.length(); ++ii) { + final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); + if (currentByteCount + bytesConsumed > maxBytes) { + splits.add(originalString.substring(previousEnd, ii)); + previousEnd = ii; + currentByteCount = 0; } - }); + currentByteCount += bytesConsumed; + } + splits.add(originalString.substring(previousEnd)); + return splits.toArray(String[]::new); + } - return wantRetry && !toRetry.isEmpty(); + private static int calcBytesConsumed(final char ch) { + if (ch == 0) { + return 2; + } + if (ch <= 0x7f) { + return 1; + } + if (ch <= 0x7ff) { + return 2; + } + return 3; } /** @@ -1331,3 +1019,5 @@ private static String getJavaClassPath() { return javaClasspath; } } + + diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index eee2d650535..dfe085504f3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -95,7 +95,7 @@ protected AbstractScriptSession( this.classCacheDirectory = classCacheDirectory; queryScope = new ScriptSessionQueryScope(); - final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory, parentClassLoader); + final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory); executionContext = ExecutionContext.newBuilder() .markSystemic() diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 6bb8c5d262d..477290ceff3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -690,12 +690,6 @@ && isAnInteger(aClass.getName().substring(SCRIPT_PREFIX.length()))) { notifiedQueryLibrary = true; executionContext.getQueryLibrary().updateVersionString(); } - - try { - QueryCompilerImpl.writeClass(classCacheDirectory, entry.getKey(), entry.getValue()); - } catch (IOException e) { - throw new RuntimeException(e); - } } } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilterGeneration.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilterGeneration.java index 9702ca6ad4a..848eb71bdb0 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilterGeneration.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilterGeneration.java @@ -4,7 +4,6 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.table.Table; import io.deephaven.engine.util.TableTools; import io.deephaven.engine.table.impl.util.ModelFileGenerator; diff --git a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java index 4cef60a63b6..08b0e0439f9 100644 --- a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java +++ b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java @@ -5,10 +5,9 @@ import io.deephaven.api.agg.Aggregation; import io.deephaven.auth.AuthContext; -import io.deephaven.configuration.DataDir; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; @@ -31,7 +30,6 @@ import org.apache.commons.lang3.mutable.MutableObject; import org.junit.*; -import java.nio.file.Path; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -114,10 +112,7 @@ public void run() { } private QueryCompiler compilerForUnitTests() { - final Path queryCompilerDir = DataDir.get() - .resolve("io.deephaven.engine.updategraph.impl.TestEventDrivenUpdateGraph.compilerForUnitTests"); - - return QueryCompilerImpl.create(queryCompilerDir.toFile(), getClass().getClassLoader()); + return QueryCompilerImpl.create(); } @Test diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index da1d636c03b..10435309133 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.create()) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index 9ddfb85baf8..dd10bb8c06e 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompilerImpl.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) + QueryCompilerImpl.create()) .build(); exContextCloseable = context.open(); diff --git a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java index dceb067d235..c3ceb18b917 100644 --- a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java +++ b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java @@ -56,8 +56,7 @@ public void before() throws IOException { cacheDir.deleteOnExit(); executionContext = ExecutionContext.newBuilder().newQueryLibrary().newQueryScope() - .setQueryCompiler(QueryCompilerImpl.create( - cacheDir, TestUpdateAncestorViz.class.getClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create()) .setOperationInitializer(ForkJoinPoolOperationInitializer.fromCommonPool()) .setUpdateGraph(defaultUpdateGraph).build().withAuthContext(new AuthContext.Anonymous()); }