From f288f4ff7e4a23a9e73a85dee59a44fb37c6a268 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 23 Jun 2026 16:14:53 -0500 Subject: [PATCH 01/12] claude-generated rewrite, avoid writing formulas to disk --- docs/groovy/conceptual/execution-context.md | 6 +- docs/python/conceptual/execution-context.md | 10 +- .../context/InMemoryQueryCompiler.java.bak | 722 +++++++++++++++ .../engine/context/QueryCompiler.java | 10 + .../engine/context/TestQueryCompiler.java | 13 +- .../engine/context/InMemoryQueryCompiler.java | 871 ++++++++++++++++++ .../table/impl/select/ConditionFilter.java | 4 +- .../table/impl/select/DhFormulaColumn.java | 6 +- .../impl/select/QueryScopeParamTypeUtil.java | 4 +- .../select/codegen/JavaKernelBuilder.java | 6 +- .../engine/util/AbstractScriptSession.java | 2 +- .../engine/util/DynamicCompileUtils.java | 6 +- .../engine/util/GroovyDeephavenSession.java | 2 +- .../select/TestConditionFilterGeneration.java | 1 - .../impl/TestEventDrivenUpdateGraph.java | 7 +- .../scripts/TestGroovyDeephavenSession.java | 4 +- .../engine/context/TestExecutionContext.java | 2 +- .../testcase/RefreshingTableTestCase.java | 6 +- .../parquet/table/TableWriteBenchmark.java | 4 +- .../impl/util/TestUpdateAncestorViz.java | 5 +- 20 files changed, 1641 insertions(+), 50 deletions(-) create mode 100644 engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak create mode 100644 engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java diff --git a/docs/groovy/conceptual/execution-context.md b/docs/groovy/conceptual/execution-context.md index 347d91ac2bb..e38d18f5e36 100644 --- a/docs/groovy/conceptual/execution-context.md +++ b/docs/groovy/conceptual/execution-context.md @@ -225,7 +225,7 @@ The following example demonstrates how to build an `ExecutionContext` from scrat ```groovy skip-test import io.deephaven.engine.context.ExecutionContext -import io.deephaven.engine.context.QueryCompilerImpl +import io.deephaven.engine.context.InMemoryQueryCompiler import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph import io.deephaven.engine.updategraph.OperationInitializer import java.nio.file.Files @@ -235,7 +235,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile(), ClassLoader.getSystemClassLoader())) + .setQueryCompiler(InMemoryQueryCompiler.create(Files.createTempDirectory("qc_").toFile())) .build() ``` @@ -262,7 +262,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(eventDrivenGraph) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile(), ClassLoader.getSystemClassLoader())) + .setQueryCompiler(InMemoryQueryCompiler.create(Files.createTempDirectory("qc_").toFile())) .build() ``` diff --git a/docs/python/conceptual/execution-context.md b/docs/python/conceptual/execution-context.md index eb7f69f6c4e..88a27aa5ebe 100644 --- a/docs/python/conceptual/execution-context.md +++ b/docs/python/conceptual/execution-context.md @@ -244,7 +244,7 @@ from deephaven.jcompat import j_hashmap import tempfile ExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") -QueryCompilerImpl = jpy.get_type("io.deephaven.engine.context.QueryCompilerImpl") +InMemoryQueryCompiler = jpy.get_type("io.deephaven.engine.context.InMemoryQueryCompiler") PeriodicUpdateGraph = jpy.get_type( "io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph" ) @@ -261,9 +261,8 @@ execution_context = ( .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) .setQueryCompiler( - QueryCompilerImpl.create( + InMemoryQueryCompiler.create( jpy.get_type("java.io.File")(temp_dir), - jpy.get_type("java.lang.ClassLoader").getSystemClassLoader(), ) ) .build() @@ -288,7 +287,7 @@ import jpy import tempfile ExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") -QueryCompilerImpl = jpy.get_type("io.deephaven.engine.context.QueryCompilerImpl") +InMemoryQueryCompiler = jpy.get_type("io.deephaven.engine.context.InMemoryQueryCompiler") EventDrivenUpdateGraph = jpy.get_type( "io.deephaven.engine.updategraph.impl.EventDrivenUpdateGraph" ) @@ -307,9 +306,8 @@ execution_context = ( .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(event_driven_graph) .setQueryCompiler( - QueryCompilerImpl.create( + InMemoryQueryCompiler.create( jpy.get_type("java.io.File")(temp_dir), - jpy.get_type("java.lang.ClassLoader").getSystemClassLoader(), ) ) .build() diff --git a/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak b/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak new file mode 100644 index 00000000000..4fc45cac53a --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak @@ -0,0 +1,722 @@ +// +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending +// +// package io.deephaven.engine.context; +// +// import io.deephaven.UncheckedDeephavenException; +// import io.deephaven.configuration.Configuration; +// import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +// import io.deephaven.internal.log.LoggerFactory; +// import io.deephaven.io.logger.Logger; +// import io.deephaven.util.ByteUtils; +// import io.deephaven.util.CompletionStageFuture; +// import io.deephaven.util.mutable.MutableInt; +// import org.apache.commons.text.StringEscapeUtils; +// import org.jetbrains.annotations.NotNull; +// import org.jetbrains.annotations.Nullable; +// +// import javax.tools.*; +// import java.io.*; +// import java.lang.reflect.Field; +// import java.net.MalformedURLException; +// import java.net.URI; +// import java.net.URL; +// import java.net.URLClassLoader; +// import java.nio.charset.StandardCharsets; +// import java.security.MessageDigest; +// import java.security.NoSuchAlgorithmException; +// import java.util.*; +// import java.util.concurrent.*; +// import java.util.concurrent.atomic.AtomicReference; +// import java.util.stream.Collectors; +// import java.util.stream.Stream; +// +///** +// * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory (no filesystem output). +// *

+// * Like the filesystem-backed {@link QueryCompilerImpl}, this implementation uses {@code java.class.path} plus an +// * optional extra class directory for resolving compilation dependencies (e.g., Groovy-generated bytecode). The key +// * difference is that compiled classes are defined directly into a classloader rather than written to disk. +// *

+// *

Key invariants

+// * +// */ +// public class InMemoryQueryCompiler implements QueryCompiler { +// +// private static final Logger log = LoggerFactory.getLogger(InMemoryQueryCompiler.class); +// +// private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; +// private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); +// private static final int MAX_CLASS_COLLISIONS = 128; +// private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; +// +// public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; +// public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; +// +// private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); +// +// private static JavaCompiler compiler; +// private static final AtomicReference standardFileManagerCache = new AtomicReference<>(); +// +// private static void ensureJavaCompiler() { +// synchronized (InMemoryQueryCompiler.class) { +// if (compiler == null) { +// compiler = ToolProvider.getSystemJavaCompiler(); +// if (compiler == null) { +// throw new UncheckedDeephavenException( +// "No Java compiler provided - are you using a JRE instead of a JDK?"); +// } +// } +// } +// } +// +// private static JavaFileManager acquireFileManager() { +// JavaFileManager fm = standardFileManagerCache.getAndSet(null); +// if (fm == null) { +// fm = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); +// } +// return fm; +// } +// +// private static void releaseFileManager(@NotNull final JavaFileManager fm) { +// if (!standardFileManagerCache.compareAndSet(null, fm)) { +// try { +// fm.close(); +// } catch (final IOException err) { +// throw new UncheckedIOException("Could not close JavaFileManager", err); +// } +// } +// } +// +// // ---- Instance state ---- +// +// private final Map>> knownClasses = new HashMap<>(); +// private final File extraClassDirectory; +// private final InMemoryClassLoader classLoader; +// private final List classNamesForAnnotationProcessing; +// +// /** +// * Creates a new InMemoryQueryCompiler. +// * +// * @param extraClassDirectory optional directory containing additional class files (e.g., Groovy bytecode) +// * @param parentClassLoader the parent class loader +// */ +// public static InMemoryQueryCompiler create( +// @Nullable final File extraClassDirectory, +// @NotNull final ClassLoader parentClassLoader) { +// return new InMemoryQueryCompiler(extraClassDirectory, parentClassLoader, null); +// } +// +// /** +// * Creates a new InMemoryQueryCompiler using the current thread's context classloader. +// * +// * @param extraClassDirectory optional extra class directory for compilation dependencies +// */ +// public static InMemoryQueryCompiler create(@Nullable final File extraClassDirectory) { +// return new InMemoryQueryCompiler(extraClassDirectory, Thread.currentThread().getContextClassLoader(), null); +// } +// +// static InMemoryQueryCompiler createForUnitTests() { +// return createForUnitTests(null); +// } +// +// static InMemoryQueryCompiler createForUnitTests(final List classNamesForAnnotationProcessing) { +// return new InMemoryQueryCompiler(null, InMemoryQueryCompiler.class.getClassLoader(), +// classNamesForAnnotationProcessing); +// } +// +// private InMemoryQueryCompiler( +// @Nullable final File extraClassDirectory, +// @NotNull final ClassLoader parentClassLoader, +// final List classNamesForAnnotationProcessing) { +// ensureJavaCompiler(); +// this.extraClassDirectory = extraClassDirectory; +// +// if (extraClassDirectory != null) { +// try { +// URL[] urls = new URL[] {extraClassDirectory.toURI().toURL()}; +// this.classLoader = new InMemoryClassLoader(urls, parentClassLoader); +// } catch (MalformedURLException e) { +// throw new UncheckedDeephavenException(e); +// } +// } else { +// this.classLoader = new InMemoryClassLoader(new URL[0], parentClassLoader); +// } +// +// this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; +// } +// +// public static boolean setLogEnabled(boolean logEnabled) { +// boolean original = InMemoryQueryCompiler.logEnabled; +// InMemoryQueryCompiler.logEnabled = logEnabled; +// return original; +// } +// +// @Override +// public void compile( +// @NotNull final QueryCompilerRequest[] requests, +// @NotNull final CompletionStageFuture.Resolver>[] resolvers) { +// if (requests.length == 0) { +// return; +// } +// if (requests.length != resolvers.length) { +// throw new IllegalArgumentException("Requests and resolvers must be the same length"); +// } +// +// // noinspection unchecked +// final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; +// +// final List newRequests = new ArrayList<>(); +// final List>> newResolvers = new ArrayList<>(); +// +// synchronized (this) { +// for (int ii = 0; ii < requests.length; ++ii) { +// final QueryCompilerRequest request = requests[ii]; +// final CompletionStageFuture.Resolver> resolver = resolvers[ii]; +// +// CompletionStageFuture> future = +// knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); +// if (future == null) { +// newRequests.add(request); +// newResolvers.add(resolver); +// future = resolver.getFuture(); +// } +// allFutures[ii] = future; +// } +// } +// +// if (!newResolvers.isEmpty()) { +// try { +// compileHelper(newRequests, newResolvers); +// } catch (RuntimeException e) { +// synchronized (this) { +// for (int ii = 0; ii < newRequests.size(); ++ii) { +// if (newResolvers.get(ii).completeExceptionally(e)) { +// knownClasses.remove(newRequests.get(ii).classBody()); +// } +// } +// } +// throw e; +// } +// } +// +// for (int ii = 0; ii < requests.length; ++ii) { +// try { +// resolvers[ii].complete(allFutures[ii].get()); +// } catch (ExecutionException err) { +// resolvers[ii].completeExceptionally(err.getCause()); +// } catch (InterruptedException err) { +// resolvers[ii].completeExceptionally(err); +// } catch (Throwable err) { +// resolvers[ii].completeExceptionally(err); +// } +// } +// } +// +// // ---- Compilation logic ---- +// +// private static class CompilationState { +// int nextProbeIndex; +// boolean complete; +// String packageName; +// String fqClassName; +// } +// +// private void compileHelper( +// @NotNull final List requests, +// @NotNull final List>> resolvers) { +// final MessageDigest digest; +// try { +// digest = MessageDigest.getInstance("SHA-256"); +// } catch (NoSuchAlgorithmException e) { +// throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); +// } +// +// final String[] basicHashText = new String[requests.size()]; +// for (int ii = 0; ii < requests.size(); ++ii) { +// basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( +// requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); +// } +// +// int numComplete = 0; +// final CompilationState[] states = new CompilationState[requests.size()]; +// for (int ii = 0; ii < requests.size(); ++ii) { +// states[ii] = new CompilationState(); +// } +// +// while (numComplete < requests.size()) { +// for (int ii = 0; ii < requests.size(); ++ii) { +// final CompilationState state = states[ii]; +// if (state.complete) { +// continue; +// } +// +// next_probe: while (true) { +// final int pi = state.nextProbeIndex++; +// final String packageNameSuffix = "c_" + basicHashText[ii] +// + (pi == 0 ? "" : ("p" + pi)) +// + "v" + JAVA_CLASS_VERSION; +// +// final QueryCompilerRequest request = requests.get(ii); +// if (pi >= MAX_CLASS_COLLISIONS) { +// Exception err = new IllegalStateException("Found too many collisions for package name root " +// + request.packageNameRoot() + ", class name=" + request.className() + ", class body " +// + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); +// resolvers.get(ii).completeExceptionally(err); +// state.complete = true; +// ++numComplete; +// break; +// } +// +// state.packageName = request.getPackageName(packageNameSuffix); +// state.fqClassName = state.packageName + "." + request.className(); +// +// for (int jj = 0; jj < ii; ++jj) { +// if (states[jj].fqClassName.equals(state.fqClassName)) { +// continue next_probe; +// } +// } +// +// Class result = tryLoadClassByFqName(state.fqClassName); +// if (result == null) { +// break; +// } +// +// if (completeIfResultMatches(state.packageName, request, resolvers.get(ii), result)) { +// state.complete = true; +// ++numComplete; +// break; +// } +// } +// } +// +// if (numComplete == requests.size()) { +// return; +// } +// +// final List attempts = new ArrayList<>(); +// for (int ii = 0; ii < requests.size(); ++ii) { +// final CompilationState state = states[ii]; +// if (!state.complete) { +// attempts.add(new CompilationRequestAttempt( +// requests.get(ii), state.packageName, state.fqClassName, resolvers.get(ii))); +// } +// } +// +// doCompileAndDefine(attempts); +// +// for (int ii = 0; ii < requests.size(); ++ii) { +// final CompilationState state = states[ii]; +// if (state.complete) { +// continue; +// } +// +// final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); +// if (resolver.getFuture().isDone()) { +// state.complete = true; +// ++numComplete; +// continue; +// } +// +// final QueryCompilerRequest request = requests.get(ii); +// Class clazz = tryLoadClassByFqName(state.fqClassName); +// if (clazz == null) { +// throw new IllegalStateException("Class not found after compilation: " + state.fqClassName); +// } +// +// if (completeIfResultMatches(state.packageName, request, resolver, clazz)) { +// state.complete = true; +// ++numComplete; +// } +// } +// } +// } +// +// private boolean completeIfResultMatches( +// final String packageName, +// final QueryCompilerRequest request, +// final CompletionStageFuture.Resolver> resolver, +// final Class result) { +// final String identifyingFieldValue = loadIdentifyingField(result); +// if (!request.classBody().equals(identifyingFieldValue)) { +// return false; +// } +// +// request.codeLog() +// .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); +// +// resolver.complete(result); +// +// synchronized (this) { +// knownClasses.remove(identifyingFieldValue); +// knownClasses.put(identifyingFieldValue, resolver.getFuture()); +// } +// +// return true; +// } +// +// private Class tryLoadClassByFqName(String fqClassName) { +// try { +// return classLoader.loadClass(fqClassName); +// } catch (ClassNotFoundException cnfe) { +// return null; +// } +// } +// +// private static String loadIdentifyingField(Class c) { +// try { +// final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); +// return (String) field.get(null); +// } catch (NoSuchFieldException | IllegalAccessException e) { +// throw new IllegalStateException("Malformed class in cache", e); +// } +// } +// +// // ---- Compilation and class definition ---- +// +// private void doCompileAndDefine(@NotNull final List requests) { +// final JavaFileManager fileManager = acquireFileManager(); +// try { +// final List toRetry = new ArrayList<>(); +// final boolean wantRetry = compileSingleRound(fileManager, requests, 0, requests.size(), toRetry); +// if (wantRetry) { +// final List ignored = new ArrayList<>(); +// if (compileSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { +// throw new IllegalStateException("Unexpected failure during second pass of compilation"); +// } +// } +// } finally { +// releaseFileManager(fileManager); +// } +// } +// +// private boolean compileSingleRound( +// @NotNull final JavaFileManager fileManager, +// @NotNull final List requests, +// final int startInclusive, +// final int endExclusive, +// @NotNull final List toRetry) { +// +// final InMemoryFileManager inMemoryFm = new InMemoryFileManager(fileManager); +// final StringWriter compilerOutput = new StringWriter(); +// +// final String classPathAsString = getClassPath(); +// final List compilerOptions = Arrays.asList( +// "-cp", classPathAsString, +// "--should-stop=ifError=GENERATE"); +// +// final MutableInt numFailures = new MutableInt(0); +// final List globalFailures = new ArrayList<>(); +// +// compiler.getTask(compilerOutput, +// inMemoryFm, +// diagnostic -> { +// if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { +// return; +// } +// +// final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); +// +// if (source == null) { +// final UncheckedDeephavenException err = new UncheckedDeephavenException( +// "Error Invoking Compiler, no source present in diagnostic:\n" +// + diagnostic.getMessage(Locale.getDefault())); +// globalFailures.add(err); +// return; +// } +// +// final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " +// + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); +// if (source.resolver.completeExceptionally(err)) { +// numFailures.increment(); +// } +// }, +// compilerOptions, +// classNamesForAnnotationProcessing, +// requests.subList(startInclusive, endExclusive).stream() +// .map(CompilationRequestAttempt::makeSource) +// .collect(Collectors.toList())) +// .call(); +// +// if (!globalFailures.isEmpty()) { +// final RuntimeException e0 = globalFailures.get(0); +// for (int ii = 1; ii < globalFailures.size(); ++ii) { +// e0.addSuppressed(globalFailures.get(ii)); +// } +// throw e0; +// } +// +// final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; +// +// final Map compiledClasses = inMemoryFm.getCompiledClasses(); +// for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { +// if (request.resolver.getFuture().isDone()) { +// continue; +// } +// +// if (!compiledClasses.containsKey(request.fqClassName)) { +// if (wantRetry) { +// toRetry.add(request); +// } +// continue; +// } +// +// final String classPrefix = request.fqClassName; +// for (Map.Entry entry : compiledClasses.entrySet()) { +// final String className = entry.getKey(); +// if (className.equals(classPrefix) || className.startsWith(classPrefix + "$")) { +// classLoader.addClass(className, entry.getValue()); +// } +// } +// } +// +// return wantRetry && !toRetry.isEmpty(); +// } +// +// // ---- Classpath ---- +// +// private String getClassPath() { +// final StringBuilder sb = new StringBuilder(System.getProperty("java.class.path")); +// if (extraClassDirectory != null) { +// sb.append(File.pathSeparator).append(extraClassDirectory.getAbsolutePath()); +// } +// +// String result = sb.toString(); +// final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; +// if (result.matches(intellijClassPathJarRegex)) { +// try { +// final Enumeration resources = +// InMemoryQueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); +// final java.util.jar.Attributes.Name createdByAttribute = +// new java.util.jar.Attributes.Name("Created-By"); +// final java.util.jar.Attributes.Name classPathAttribute = +// new java.util.jar.Attributes.Name("Class-Path"); +// while (resources.hasMoreElements()) { +// final java.util.jar.Manifest manifest = +// new java.util.jar.Manifest(resources.nextElement().openStream()); +// final java.util.jar.Attributes attributes = manifest.getMainAttributes(); +// final Object createdBy = attributes.get(createdByAttribute); +// if ("IntelliJ IDEA".equals(createdBy)) { +// final String extendedClassPath = (String) attributes.get(classPathAttribute); +// if (extendedClassPath != null) { +// final String filePaths = Stream.of(extendedClassPath.split("file:/")) +// .map(String::trim) +// .filter(fileName -> !fileName.isEmpty()) +// .collect(Collectors.joining(File.pathSeparator)); +// result = Stream.of(result.split(File.pathSeparator)) +// .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) +// .collect(Collectors.joining(File.pathSeparator)); +// } +// } +// } +// } catch (IOException e) { +// throw new UncheckedIOException("Error extracting manifest file from classpath.\n", e); +// } +// } +// return result; +// } +// +// // ---- InMemoryClassLoader ---- +// +// /** +// * A URLClassLoader that also supports defining classes from in-memory bytecode. +// * The URL[] includes the extra class directory so parent delegation finds Groovy classes. +// */ +// private static class InMemoryClassLoader extends URLClassLoader { +// +// private final ConcurrentHashMap inMemoryClasses = new ConcurrentHashMap<>(); +// +// InMemoryClassLoader(URL[] urls, ClassLoader parent) { +// super(urls, parent); +// } +// +// void addClass(String name, byte[] bytes) { +// inMemoryClasses.put(name, bytes); +// } +// +// @Override +// protected Class findClass(String name) throws ClassNotFoundException { +// final byte[] bytes = inMemoryClasses.remove(name); +// if (bytes != null) { +// return defineClass(name, bytes, 0, bytes.length); +// } +// return super.findClass(name); +// } +// } +// +// // ---- InMemoryFileManager ---- +// +// /** +// * A forwarding JavaFileManager that intercepts class output, writing to in-memory byte arrays. +// */ +// private static class InMemoryFileManager extends ForwardingJavaFileManager { +// +// private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); +// +// InMemoryFileManager(JavaFileManager delegate) { +// super(delegate); +// } +// +// Map getCompiledClasses() { +// final Map result = new HashMap<>(); +// for (Map.Entry entry : outputClasses.entrySet()) { +// result.put(entry.getKey(), entry.getValue().getBytes()); +// } +// return result; +// } +// +// @Override +// public JavaFileObject getJavaFileForOutput( +// Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { +// final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); +// outputClasses.put(className, fileObject); +// return fileObject; +// } +// } +// +// // ---- InMemoryClassFileObject ---- +// +// private static class InMemoryClassFileObject extends SimpleJavaFileObject { +// private final String className; +// private ByteArrayOutputStream outputStream; +// +// InMemoryClassFileObject(String className) { +// super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); +// this.className = className; +// } +// +// @Override +// public OutputStream openOutputStream() { +// outputStream = new ByteArrayOutputStream(); +// return outputStream; +// } +// +// byte[] getBytes() { +// if (outputStream == null) { +// throw new IllegalStateException("No bytes available for " + className); +// } +// return outputStream.toByteArray(); +// } +// } +// +// // ---- Source representation ---- +// +// private static class JavaSourceFromString extends SimpleJavaFileObject { +// final String description; +// final String code; +// final CompletionStageFuture.Resolver> resolver; +// +// JavaSourceFromString(String description, String name, String code, +// CompletionStageFuture.Resolver> resolver) { +// super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); +// this.description = description; +// this.code = code; +// this.resolver = resolver; +// } +// +// @Override +// public CharSequence getCharContent(boolean ignoreEncodingErrors) { +// return code; +// } +// } +// +// private static class CompilationRequestAttempt { +// final String description; +// final String fqClassName; +// final String finalCode; +// final String packageName; +// final QueryCompilerRequest request; +// final CompletionStageFuture.Resolver> resolver; +// +// CompilationRequestAttempt( +// @NotNull final QueryCompilerRequest request, +// @NotNull final String packageName, +// @NotNull final String fqClassName, +// @NotNull final CompletionStageFuture.Resolver> resolver) { +// this.description = request.description(); +// this.fqClassName = fqClassName; +// this.resolver = resolver; +// this.packageName = packageName; +// this.request = request; +// this.finalCode = makeFinalCode(request.className(), request.classBody(), packageName); +// +// if (logEnabled) { +// log.info().append("Generating code ").append(finalCode).endl(); +// } +// } +// +// JavaSourceFromString makeSource() { +// return new JavaSourceFromString(description, fqClassName, finalCode, resolver); +// } +// } +// +// // ---- Code generation utilities ---- +// +// static String makeFinalCode(String className, String classBody, String packageName) { +// if (classBody.contains("$CLASSNAME$")) { +// throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" +// + " the final class name affects the compiled byte code and therefore cannot be dynamically " +// + "replaced."); +// } +// +// final String joinedEscapedBody = createEscapedJoinedString(classBody); +// classBody = classBody.substring(0, classBody.lastIndexOf("}")); +// classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; +// return "package " + packageName + ";\n" + classBody; +// } +// +// public static String createEscapedJoinedString(final String originalString) { +// return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); +// } +// +// public static String createEscapedJoinedString(final String originalString, int maxStringLength) { +// final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); +// for (int ii = 0; ii < splits.length; ++ii) { +// final String escaped = StringEscapeUtils.escapeJava(splits[ii]); +// splits[ii] = "\"" + escaped + "\""; +// } +// if (splits.length == 1) { +// return splits[0]; +// } +// final String formattedInnards = String.join(",\n", splits); +// return "String.join(\"\", " + formattedInnards + ")"; +// } +// +// private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { +// final List splits = new ArrayList<>(); +// int previousEnd = 0; +// int currentByteCount = 0; +// for (int ii = 0; ii < originalString.length(); ++ii) { +// final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); +// if (currentByteCount + bytesConsumed > maxBytes) { +// splits.add(originalString.substring(previousEnd, ii)); +// previousEnd = ii; +// currentByteCount = 0; +// } +// currentByteCount += bytesConsumed; +// } +// splits.add(originalString.substring(previousEnd)); +// return splits.toArray(String[]::new); +// } +// +// private static int calcBytesConsumed(final char ch) { +// if (ch == 0) { +// return 2; +// } +// if (ch <= 0x7f) { +// return 1; +// } +// if (ch <= 0x7ff) { +// return 2; +// } +// return 3; +// } +// +// } +// diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 4041f91cf02..f9adfc895a4 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -10,6 +10,16 @@ import java.util.concurrent.ExecutionException; +/** + * Java compiler api, specifically geared towards query language expressions. Implementations should be constructed with + * a path on disk to existing bytecode to use in addition to `java.class.path`. Likewise, callers must ensure that the + * context classloader already contains those input classes - the new class will be loaded in a child classloader to + * ensure that the class can be GCd when neither it nor the compiler instance are reachable. + *

+ * Requests for compilation will never depend on other classes compiled by this or any other QueryCompiler, and are + * expected to be unique based on the provided classname. This enables the implementation to cache classes and only + * compile and load a class once per name. + */ public interface QueryCompiler { /** diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index fa39ad0211f..4a4e389dfe9 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -14,9 +14,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import java.io.IOException; import java.lang.reflect.Method; import java.time.Instant; import java.time.LocalDateTime; @@ -60,18 +58,14 @@ private static String generateClassBody(final String className) { @Rule public final EngineCleanup framework = new EngineCleanup(); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - private SafeCloseable executionContextClosable; @Before - public void setUp() throws IOException { + public void setUp() { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(QueryCompilerImpl.create( - folder.newFolder(), TestQueryCompiler.class.getClassLoader())) + .setQueryCompiler(InMemoryQueryCompiler.create(null)) .build() .open(); } @@ -375,7 +369,8 @@ public void testBadCompile() { CompletionStageFuture.make(), }; - final QueryCompilerImpl badCompiler = QueryCompilerImpl.createForUnitTests(List.of("InvalidClassArgument")); + final InMemoryQueryCompiler badCompiler = + InMemoryQueryCompiler.createForUnitTests(List.of("InvalidClassArgument")); UncheckedDeephavenException e = org.junit.Assert.assertThrows(UncheckedDeephavenException.class, () -> badCompiler.compile(requests, resolvers)); org.junit.Assert.assertEquals("Error Invoking Compiler, no source present in diagnostic:\n" + diff --git a/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java b/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java new file mode 100644 index 00000000000..f4a2fce693b --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java @@ -0,0 +1,871 @@ +// +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.context; + +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.log.LogOutput; +import io.deephaven.base.log.LogOutputAppendable; +import io.deephaven.base.verify.Assert; +import io.deephaven.configuration.Configuration; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.engine.table.impl.util.ImmediateJobScheduler; +import io.deephaven.engine.table.impl.util.JobScheduler; +import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.log.impl.LogOutputStringImpl; +import io.deephaven.io.logger.Logger; +import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.mutable.MutableInt; +import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.tools.*; +import java.io.*; +import java.lang.reflect.Field; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory and defines the resulting + * classes via {@link ClassLoader#defineClass}, avoiding filesystem writes for compilation output. + * + *

+ * The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory + * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the + * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Class nor this + * compiler instance are reachable. + * + *

Constraints

+ *
    + *
  • Compiled classes must not depend on other classes compiled by this or any other QueryCompiler instance.
    • + *
  • The caller must ensure the parent classloader already contains any classes referenced by compiled formulas (e.g., + * Groovy-defined classes).
    • + *
  • Each compilation request produces a single top-level class (matching {@link QueryCompilerRequest#className()}) + * which may contain static inner classes and anonymous classes.
    • + *
+ */ +public class InMemoryQueryCompiler implements QueryCompiler, LogOutputAppendable { + + private static final Logger log = LoggerFactory.getLogger(InMemoryQueryCompiler.class); + + private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; + private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); + private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; + + private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); + + public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; + public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; + + // --- Static compiler state --- + + private static JavaCompiler compiler; + private static final AtomicReference fileManagerCache = new AtomicReference<>(); + + private static void ensureJavaCompiler() { + synchronized (InMemoryQueryCompiler.class) { + if (compiler == null) { + compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new UncheckedDeephavenException( + "No Java compiler provided - are you using a JRE instead of a JDK?"); + } + } + } + } + + private static JavaFileManager acquireFileManager() { + JavaFileManager fileManager = fileManagerCache.getAndSet(null); + if (fileManager == null) { + fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); + } + return fileManager; + } + + private static void releaseFileManager(@NotNull final JavaFileManager fileManager) { + if (!fileManagerCache.compareAndSet(null, fileManager)) { + try { + fileManager.close(); + } catch (final IOException err) { + throw new UncheckedIOException("Could not close JavaFileManager", err); + } + } + } + + // --- Instance state --- + + /** Deduplication map: classBody → future holding the compiled Class. */ + private final Map>> knownClasses = new HashMap<>(); + + /** Set of fully-qualified class names already assigned, for collision avoidance. */ + private final Set takenNames = new HashSet<>(); + + /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ + private final ClassLoader parentClassLoader; + + /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ + @Nullable + private final File additionalClassPathDir; + + /** For test use only: specifying a non-null list causes an annotation processing error. */ + private final List classNamesForAnnotationProcessing; + + // --- Factory methods --- + + /** + * Creates a new InMemoryQueryCompiler. Uses the current thread's context classloader as the parent for per-batch + * classloaders (as required by {@link QueryCompiler}'s contract). + * + * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) + */ + public static InMemoryQueryCompiler create(@Nullable final File additionalClassPathDir) { + return new InMemoryQueryCompiler(additionalClassPathDir, null); + } + + static InMemoryQueryCompiler createForUnitTests() { + return createForUnitTests(null); + } + + static InMemoryQueryCompiler createForUnitTests(final List classNamesForAnnotationProcessing) { + return new InMemoryQueryCompiler(null, classNamesForAnnotationProcessing); + } + + private InMemoryQueryCompiler( + @Nullable final File additionalClassPathDir, + final List classNamesForAnnotationProcessing) { + ensureJavaCompiler(); + this.additionalClassPathDir = additionalClassPathDir; + this.parentClassLoader = Thread.currentThread().getContextClassLoader(); + this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; + } + + // --- Public API --- + + @Override + public LogOutput append(LogOutput logOutput) { + return logOutput.append("InMemoryQueryCompiler{additionalClassPathDir=") + .append(additionalClassPathDir == null ? "null" : additionalClassPathDir.getAbsolutePath()) + .append("}"); + } + + @Override + public String toString() { + return new LogOutputStringImpl().append(this).toString(); + } + + /** + * Enables or disables compilation logging. + * + * @param logEnabled Whether logging should be enabled + * @return The value of {@code logEnabled} before calling this method. + */ + public static boolean setLogEnabled(boolean logEnabled) { + boolean original = InMemoryQueryCompiler.logEnabled; + InMemoryQueryCompiler.logEnabled = logEnabled; + return original; + } + + @Override + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); + + synchronized (this) { + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; + } + } + + if (!newResolvers.isEmpty()) { + log.info().append("InMemoryQueryCompiler.compile: compiling ").append(newResolvers.size()) + .append(" new requests out of ").append(requests.length).append(" total").endl(); + try { + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + log.error().append("InMemoryQueryCompiler.compile: compileHelper threw ").append(e.toString()).endl(); + synchronized (this) { + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } + } + } + throw e; + } + log.info().append("InMemoryQueryCompiler.compile: compileHelper completed successfully").endl(); + } else { + log.info().append("InMemoryQueryCompiler.compile: all ").append(requests.length) + .append(" requests resolved from cache").endl(); + } + + for (int ii = 0; ii < requests.length; ++ii) { + try { + final CompletionStageFuture> future = allFutures[ii]; + if (!future.isDone()) { + log.info().append("InMemoryQueryCompiler: waiting for future[").append(ii) + .append("] className=").append(requests[ii].className()) + .append(" classBody hash=").append( + Integer.toHexString(requests[ii].classBody().hashCode())) + .append(" future=").append(future.toString()) + .endl(); + } + final Class result = future.get(10, TimeUnit.SECONDS); + resolvers[ii].complete(result); + } catch (TimeoutException err) { + final String msg = "InMemoryQueryCompiler: Timed out (10s) waiting for class compilation" + + " request[" + ii + "] className=" + requests[ii].className() + + " future=" + allFutures[ii] + + " isDone=" + allFutures[ii].isDone(); + log.error().append(msg).endl(); + // Fail all remaining resolvers and throw immediately + final UncheckedDeephavenException timeout = new UncheckedDeephavenException(msg); + for (int jj = ii; jj < requests.length; ++jj) { + resolvers[jj].completeExceptionally(timeout); + } + throw timeout; + } catch (ExecutionException err) { + resolvers[ii].completeExceptionally(err.getCause()); + } catch (InterruptedException err) { + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + resolvers[ii].completeExceptionally(err); + } catch (Throwable err) { + resolvers[ii].completeExceptionally(err); + } + } + log.info().append("InMemoryQueryCompiler.compile: returning, all ").append(requests.length) + .append(" resolvers processed").endl(); + } + + // --- Compilation logic --- + + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); + } + + // Assign unique FQ class names for each request + final String[] fqClassNames = new String[requests.size()]; + final String[] packageNames = new String[requests.size()]; + + synchronized (this) { + for (int ii = 0; ii < requests.size(); ++ii) { + final QueryCompilerRequest request = requests.get(ii); + final String hashText = ByteUtils.byteArrToHex(digest.digest( + request.classBody().getBytes(StandardCharsets.UTF_8))); + + String fqClassName = null; + String packageName = null; + for (int pi = 0; pi < 128; ++pi) { + final String packageNameSuffix = "c_" + hashText + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + packageName = request.getPackageName(packageNameSuffix); + fqClassName = packageName + "." + request.className(); + if (!takenNames.contains(fqClassName)) { + break; + } + fqClassName = null; + } + if (fqClassName == null) { + resolvers.get(ii).completeExceptionally(new IllegalStateException( + "Unable to assign unique class name for " + request.className())); + continue; + } + takenNames.add(fqClassName); + fqClassNames[ii] = fqClassName; + packageNames[ii] = packageName; + } + } + + // Build compilation attempts for requests that got a valid name + final List attempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + if (fqClassNames[ii] == null) { + continue; // already failed + } + attempts.add(new CompilationRequestAttempt( + requests.get(ii), packageNames[ii], fqClassNames[ii], resolvers.get(ii))); + } + + if (attempts.isEmpty()) { + return; + } + + // Compile and define + compileAndDefine(attempts); + + // Validate _CLASS_BODY_ field on successfully defined classes + for (int ii = 0; ii < requests.size(); ++ii) { + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + continue; // already completed (success or failure) + } + // This shouldn't happen - compileAndDefine should have resolved everything + resolver.completeExceptionally(new IllegalStateException( + "Class was not resolved after compilation: " + fqClassNames[ii])); + } + } + + private void compileAndDefine(@NotNull final List requests) { + final ExecutionContext executionContext = ExecutionContext.getContext(); + final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); + + final int numTasks; + final JobScheduler jobScheduler; + + final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); + if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { + numTasks = 1; + jobScheduler = new ImmediateJobScheduler(); + } else { + numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + jobScheduler = new OperationInitializerJobScheduler(); + } + + final JavaFileManager fileManager = acquireFileManager(); + final AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + final Runnable cleanup = () -> { + try { + releaseFileManager(fileManager); + } catch (Exception e) { + // ignore + } finally { + latch.countDown(); + } + }; + + final Consumer onError = err -> { + if (err instanceof RuntimeException) { + exception.set((RuntimeException) err); + } else { + exception.set(new UncheckedDeephavenException("Error during compilation", err)); + } + cleanup.run(); + }; + + jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, + 0, numTasks, (context, jobId, nestedErrorConsumer) -> { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + doCompileAndDefine(fileManager, requests, startInclusive, endExclusive); + }, + () -> { + }, + cleanup, + onError); + + try { + log.info().append("InMemoryQueryCompiler.compileAndDefine: awaiting latch for ").append(numTasks) + .append(" task(s), ").append(requests.size()).append(" request(s), canParallelize=") + .append(canParallelize).endl(); + final boolean completed = latch.await(10, TimeUnit.SECONDS); + if (!completed) { + final String msg = "InMemoryQueryCompiler.compileAndDefine: latch timed out after 10s!" + + " numTasks=" + numTasks + " requests=" + requests.size(); + log.error().append(msg).endl(); + throw new UncheckedDeephavenException(msg); + } + log.info().append("InMemoryQueryCompiler.compileAndDefine: latch completed").endl(); + final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); + if (perfEntry != null) { + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); + } + final RuntimeException err = exception.get(); + if (err != null) { + throw err; + } + } catch (final InterruptedException e) { + throw new CancellationException("interrupted while compiling"); + } + } + + private void doCompileAndDefine( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + final boolean wantRetry = doCompileAndDefineSingleRound( + fileManager, requests, startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; + } + // Retry non-failing requests from the first pass + final List ignored = new ArrayList<>(); + if (doCompileAndDefineSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { + throw new IllegalStateException("Unexpected failure during second pass of compilation"); + } + } + + private boolean doCompileAndDefineSingleRound( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive, + @NotNull final List toRetry) { + + // Create an in-memory file manager that captures compiled output + final InMemoryOutputFileManager outputFm = new InMemoryOutputFileManager(fileManager); + final StringWriter compilerOutput = new StringWriter(); + + final String classPathAsString = getClassPath(); + final List compilerOptions = Arrays.asList( + "-cp", classPathAsString, + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + final List globalFailures = new ArrayList<>(); + + compiler.getTask(compilerOutput, + outputFm, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + + if (source == null) { + final UncheckedDeephavenException err = new UncheckedDeephavenException( + "Error Invoking Compiler, no source present in diagnostic:\n" + + diagnostic.getMessage(Locale.getDefault())); + globalFailures.add(err); + return; + } + + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + numFailures.increment(); + } + }, + compilerOptions, + classNamesForAnnotationProcessing, + requests.subList(startInclusive, endExclusive).stream() + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final String compilerOutputText = compilerOutput.toString(); + if (!compilerOutputText.isEmpty()) { + log.info().append("Compiler output:\n").append(compilerOutputText).endl(); + } + + if (!globalFailures.isEmpty()) { + final RuntimeException e0 = globalFailures.get(0); + for (int ii = 1; ii < globalFailures.size(); ++ii) { + e0.addSuppressed(globalFailures.get(ii)); + } + throw e0; + } + + final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; + + // Define compiled classes into a per-batch classloader + final Map compiledClasses = outputFm.getCompiledClasses(); + log.info().append("InMemoryQueryCompiler: compilation produced ").append(compiledClasses.size()) + .append(" classes, numFailures=").append(numFailures.get()) + .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") + .endl(); + if (!compiledClasses.isEmpty()) { + // Use the current thread's context classloader as parent so that Groovy-defined classes + // (which may have been loaded after this InMemoryQueryCompiler was constructed) are visible. + final ClassLoader currentContextCl = Thread.currentThread().getContextClassLoader(); + final ClassLoader batchParent = currentContextCl != null ? currentContextCl : parentClassLoader; + final BatchClassLoader batchCl = new BatchClassLoader(batchParent, compiledClasses); + + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (request.resolver.getFuture().isDone()) { + // already failed + continue; + } + + if (!compiledClasses.containsKey(request.fqClassName)) { + if (wantRetry) { + toRetry.add(request); + } + continue; + } + + // Load the top-level class (which triggers loading of inner/anonymous classes as needed) + final Class clazz; + try { + clazz = batchCl.loadClass(request.fqClassName); + } catch (ClassNotFoundException e) { + request.resolver.completeExceptionally(new UncheckedDeephavenException( + "Failed to load compiled class: " + request.fqClassName, e)); + continue; + } + + // Validate the identifying field + final String identifyingFieldValue = loadIdentifyingField(clazz); + if (!request.request.classBody().equals(identifyingFieldValue)) { + request.resolver.completeExceptionally(new IllegalStateException( + "Compiled class body validation failed for " + request.fqClassName)); + continue; + } + + // Notify caller with codeLog if requested + request.request.codeLog().ifPresent( + sb -> sb.append(makeFinalCode( + request.request.className(), request.request.classBody(), request.packageName))); + + // Complete the future + log.info().append("InMemoryQueryCompiler: resolving ").append(request.fqClassName).endl(); + request.resolver.complete(clazz); + + // Canonicalize the knownClasses entry + synchronized (this) { + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, request.resolver.getFuture()); + } + } + } else { + // No output at all - if there are non-failed requests, they need retry + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (!request.resolver.getFuture().isDone() && wantRetry) { + toRetry.add(request); + } + } + } + + return wantRetry && !toRetry.isEmpty(); + } + + // --- Classpath construction --- + + private String getClassPath() { + final StringBuilder sb = new StringBuilder(getJavaClassPath()); + if (additionalClassPathDir != null) { + sb.append(File.pathSeparator).append(additionalClassPathDir.getAbsolutePath()); + } + return sb.toString(); + } + + // --- BatchClassLoader --- + + /** + * A classloader that defines classes from a map of name→bytes. All classes from a single compilation batch are + * loaded together. The parent classloader handles all other class resolution. + */ + private static class BatchClassLoader extends ClassLoader { + private final Map classBytes; + + BatchClassLoader(@NotNull ClassLoader parent, @NotNull Map classBytes) { + super(parent); + this.classBytes = classBytes; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + final byte[] bytes = classBytes.get(name); + if (bytes != null) { + return defineClass(name, bytes, 0, bytes.length); + } + throw new ClassNotFoundException(name); + } + } + + // --- InMemoryOutputFileManager --- + + /** + * A forwarding JavaFileManager that intercepts class output, storing compiled bytecode in memory rather than + * writing to the filesystem. All other operations are delegated. + */ + private static class InMemoryOutputFileManager extends ForwardingJavaFileManager { + private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); + + InMemoryOutputFileManager(JavaFileManager delegate) { + super(delegate); + } + + Map getCompiledClasses() { + final Map result = new HashMap<>(); + for (Map.Entry entry : outputClasses.entrySet()) { + result.put(entry.getKey(), entry.getValue().getBytes()); + } + return result; + } + + @Override + public JavaFileObject getJavaFileForOutput( + Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { + final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); + outputClasses.put(className, fileObject); + return fileObject; + } + + @Override + public boolean hasLocation(Location location) { + if (location == StandardLocation.CLASS_OUTPUT) { + return true; + } + return super.hasLocation(location); + } + } + + // --- InMemoryClassFileObject --- + + private static class InMemoryClassFileObject extends SimpleJavaFileObject { + private final String className; + private ByteArrayOutputStream outputStream; + + InMemoryClassFileObject(String className) { + super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); + this.className = className; + } + + @Override + public OutputStream openOutputStream() { + outputStream = new ByteArrayOutputStream(); + return outputStream; + } + + byte[] getBytes() { + if (outputStream == null) { + throw new IllegalStateException("No bytes available for " + className); + } + return outputStream.toByteArray(); + } + } + + // --- Source representation --- + + private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; + final String code; + final CompletionStageFuture.Resolver> resolver; + + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; + this.code = code; + this.resolver = resolver; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } + } + + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); + } + } + + JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); + } + } + + // --- Utilities --- + + private static String loadIdentifyingField(Class c) { + try { + final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); + return (String) field.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Malformed class in cache", e); + } + } + + static String makeFinalCode(String className, String classBody, String packageName) { + if (classBody.contains("$CLASSNAME$")) { + throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" + + " the final class name affects the compiled byte code and therefore cannot be dynamically " + + "replaced."); + } + + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; + } + + /** + * Transform a string into the corresponding Java source code that compiles into that string. + */ + public static String createEscapedJoinedString(final String originalString) { + return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + } + + public static String createEscapedJoinedString(final String originalString, int maxStringLength) { + final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); + + for (int ii = 0; ii < splits.length; ++ii) { + final String escaped = StringEscapeUtils.escapeJava(splits[ii]); + splits[ii] = "\"" + escaped + "\""; + } + assert splits.length > 0; + if (splits.length == 1) { + return splits[0]; + } + final String formattedInnards = String.join(",\n", splits); + return "String.join(\"\", " + formattedInnards + ")"; + } + + private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { + final List splits = new ArrayList<>(); + int previousEnd = 0; + int currentByteCount = 0; + for (int ii = 0; ii < originalString.length(); ++ii) { + final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); + if (currentByteCount + bytesConsumed > maxBytes) { + splits.add(originalString.substring(previousEnd, ii)); + previousEnd = ii; + currentByteCount = 0; + } + currentByteCount += bytesConsumed; + } + splits.add(originalString.substring(previousEnd)); + return splits.toArray(String[]::new); + } + + private static int calcBytesConsumed(final char ch) { + if (ch == 0) { + return 2; + } + if (ch <= 0x7f) { + return 1; + } + if (ch <= 0x7ff) { + return 2; + } + return 3; + } + + /** + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables + */ + private static String getJavaClassPath() { + String javaClasspath; + { + final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); + + final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); + if (teamCityWorkDir != null) { + final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + } + javaClasspath = javaClasspathBuilder.toString(); + } + + final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; + if (javaClasspath.matches(intellijClassPathJarRegex)) { + try { + final Enumeration resources = + InMemoryQueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); + final java.util.jar.Attributes.Name createdByAttribute = + new java.util.jar.Attributes.Name("Created-By"); + final java.util.jar.Attributes.Name classPathAttribute = + new java.util.jar.Attributes.Name("Class-Path"); + while (resources.hasMoreElements()) { + final java.util.jar.Manifest manifest = + new java.util.jar.Manifest(resources.nextElement().openStream()); + final java.util.jar.Attributes attributes = manifest.getMainAttributes(); + final Object createdBy = attributes.get(createdByAttribute); + if ("IntelliJ IDEA".equals(createdBy)) { + final String extendedClassPath = (String) attributes.get(classPathAttribute); + if (extendedClassPath != null) { + final String filePaths = Stream.of(extendedClassPath.split("file:/")) + .map(String::trim) + .filter(fileName -> !fileName.isEmpty()) + .collect(Collectors.joining(File.pathSeparator)); + + javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) + .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) + .collect(Collectors.joining(File.pathSeparator)); + } + } + } + } catch (IOException e) { + throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); + } + } + return javaClasspath; + } +} + + diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 291b97362ab..7b50edb5dd7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; @@ -478,7 +478,7 @@ protected void generateFilterCode( .description("Filter Expression: " + formula) .className(CLASS_NAME) .classBody(this.classBody) - .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index 7a4c4174d32..d8ccb78e2a0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -8,7 +8,7 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; @@ -367,7 +367,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = InMemoryQueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -817,7 +817,7 @@ private void compileFormula(@NotNull final QueryCompilerRequestProcessor compila .description("Formula Expression: " + formulaString) .className(FORMULA_CLASS_NAME) .classBody(classBody) - .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()).thenApply(clazz -> { try { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java index 54d4ac2f837..f7571545c4e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java @@ -5,7 +5,7 @@ import groovy.lang.Closure; import groovy.lang.GroovyObject; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.util.type.TypeUtils; import java.lang.reflect.Modifier; @@ -51,7 +51,7 @@ private static void visitParameterClass(final Map> found, Class } final String name = cls.getName(); - if (!name.startsWith(QueryCompilerImpl.DYNAMIC_CLASS_PREFIX)) { + if (!name.startsWith(InMemoryQueryCompiler.DYNAMIC_CLASS_PREFIX)) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index 4ea80198ead..31fc3c6678d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -4,7 +4,7 @@ package io.deephaven.engine.table.impl.select.codegen; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.CompletionStageFuture; @@ -50,7 +50,7 @@ public static CompletionStageFuture create( .description("FormulaKernel: " + originalFormulaString) .className(CLASS_NAME) .classBody(classBody) - .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) .build()).thenApply(clazz -> { final FormulaKernelFactory fkf; try { @@ -242,7 +242,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = InMemoryQueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index eee2d650535..8e7fcab31eb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -95,7 +95,7 @@ protected AbstractScriptSession( this.classCacheDirectory = classCacheDirectory; queryScope = new ScriptSessionQueryScope(); - final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory, parentClassLoader); + final QueryCompiler compilerContext = InMemoryQueryCompiler.create(classCacheDirectory); executionContext = ExecutionContext.newBuilder() .markSystemic() diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index 6bdeeecd5c9..c7a6469300e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -4,7 +4,7 @@ package io.deephaven.engine.util; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; import java.util.*; @@ -60,7 +60,7 @@ public static Supplier compileSimpleFunction(final Class res .description("Simple Function: " + code) .className(className) .classBody(classBody.toString()) - .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) .build()); try { @@ -86,7 +86,7 @@ public static Class getClassThroughCompilation(final String object) { .description("Formula: return " + object + ".class") .className(className) .classBody(classBody.toString()) - .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) .build()); try { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 6bb8c5d262d..d163ff6ee74 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -81,7 +81,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession strValues = stringCol("StrValue", "hello", "world"); final Table source = TableTools.newTable(strValues); - QueryCompilerImpl.setLogEnabled(true); + InMemoryQueryCompiler.setLogEnabled(true); final Table updated = source.update("P=pred.test(StrValue)"); assertTableEquals(TableTools.newTable(strValues, booleanCol("P", true, false)), updated); final Table filtered = source.where("pred.test(StrValue)"); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index da1d636c03b..9269fa69633 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) + .setQueryCompiler(InMemoryQueryCompiler.create(null)) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java index fa68c7d56cb..d16178a6bf0 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.util.pools.ChunkPoolReleaseTracking; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.TestExecutionContext; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -86,7 +86,7 @@ public void setUp() throws Exception { delayedErrors.clear(); livenessScopeCloseable = LivenessScopeStack.open(new LivenessScope(true), true); - oldLogEnabled = QueryCompilerImpl.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); + oldLogEnabled = InMemoryQueryCompiler.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); oldSerialSafe = updateGraph.setSerialTableOperationsSafe(true); AsyncErrorLogger.init(); ChunkPoolReleaseTracking.enableStrict(); @@ -127,7 +127,7 @@ public void tearDown() throws Exception { } final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.setSerialTableOperationsSafe(oldSerialSafe); - QueryCompilerImpl.setLogEnabled(oldLogEnabled); + InMemoryQueryCompiler.setLogEnabled(oldLogEnabled); // reset the execution context executionContext.close(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index 9ddfb85baf8..a1d24d57935 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -5,7 +5,7 @@ import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.Table; @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompilerImpl.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) + InMemoryQueryCompiler.create(null)) .build(); exContextCloseable = context.open(); diff --git a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java index dceb067d235..f50446f12af 100644 --- a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java +++ b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java @@ -9,7 +9,7 @@ import io.deephaven.auth.AuthContext; import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.InMemoryQueryCompiler; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.liveness.StandaloneLivenessManager; import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfLong; @@ -56,8 +56,7 @@ public void before() throws IOException { cacheDir.deleteOnExit(); executionContext = ExecutionContext.newBuilder().newQueryLibrary().newQueryScope() - .setQueryCompiler(QueryCompilerImpl.create( - cacheDir, TestUpdateAncestorViz.class.getClassLoader())) + .setQueryCompiler(InMemoryQueryCompiler.create(null)) .setOperationInitializer(ForkJoinPoolOperationInitializer.fromCommonPool()) .setUpdateGraph(defaultUpdateGraph).build().withAuthContext(new AuthContext.Anonymous()); } From 6fe31461156ba8aab3690f9a3bb7ca770a777dc1 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 24 Jun 2026 07:47:46 -0500 Subject: [PATCH 02/12] restore clodded up comments, ordering, logging --- .../engine/context/TestQueryCompiler.java | 2 +- .../engine/context/InMemoryQueryCompiler.java | 184 ++++++++++++------ .../impl/TestEventDrivenUpdateGraph.java | 2 +- 3 files changed, 125 insertions(+), 63 deletions(-) diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 4a4e389dfe9..ca0a53170c0 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -65,7 +65,7 @@ public void setUp() { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(InMemoryQueryCompiler.create(null)) + .setQueryCompiler(InMemoryQueryCompiler.createForUnitTests()) .build() .open(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java b/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java index f4a2fce693b..fe9a1728762 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java @@ -4,6 +4,7 @@ package io.deephaven.engine.context; import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.FileUtils; import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; @@ -38,6 +39,8 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.jar.Attributes; +import java.util.jar.Manifest; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -63,17 +66,51 @@ public class InMemoryQueryCompiler implements QueryCompiler, LogOutputAppendable { private static final Logger log = LoggerFactory.getLogger(InMemoryQueryCompiler.class); - + /** + * We pick a number just shy of 65536, leaving a little elbow room for good luck. + */ private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; + private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); + private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); - public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; - public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; + private static final String TRACE_PREFIX_PROPERTY = "QueryCompiler.tracePrefixes"; + private static final String TRACE_EXCLUDE_PREFIX_PROPERTY = "QueryCompiler.excludeTracePrefixes"; + private static final Set TRACE_INCLUDE_PREFIXES = computeTracePackages(TRACE_PREFIX_PROPERTY); + private static final Set TRACE_EXCLUDE_PREFIXES = computeTracePackages(TRACE_EXCLUDE_PREFIX_PROPERTY); + + private static Set computeTracePackages(final String property) { + if (!Configuration.getInstance().hasProperty(property)) { + return Collections.emptySet(); + } + final String propertyValue = Configuration.getInstance().getProperty(property); + return Arrays.stream(propertyValue.split(",")).map(String::trim).collect(Collectors.toSet()); + } + + /** + * Should this class (or package) be traced? Even with only trace logging, we may not want to flood the log with + * "uninteresting" classes, so we provide an inclusion and an exclusion list. + * + *

+ * Excludes take precedence over includes. + *

+ * + * @param className the class/package name to check against our trace prefixes + * @return if this class/package should be traced + */ + private static boolean shouldTrace(String className) { + if (!log.isTraceEnabled()) { + return false; + } + if (TRACE_EXCLUDE_PREFIXES.stream().anyMatch(className::startsWith)) { + return false; + } + return TRACE_INCLUDE_PREFIXES.stream().anyMatch(className::startsWith); + } - // --- Static compiler state --- private static JavaCompiler compiler; private static final AtomicReference fileManagerCache = new AtomicReference<>(); @@ -99,6 +136,8 @@ private static JavaFileManager acquireFileManager() { } private static void releaseFileManager(@NotNull final JavaFileManager fileManager) { + // Reusing the file manager saves a lot of the time in the compilation process. However, we need to be careful + // to avoid keeping too many file handles open so we'll limit ourselves to just one outstanding file manager. if (!fileManagerCache.compareAndSet(null, fileManager)) { try { fileManager.close(); @@ -108,25 +147,8 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage } } - // --- Instance state --- - - /** Deduplication map: classBody → future holding the compiled Class. */ - private final Map>> knownClasses = new HashMap<>(); - - /** Set of fully-qualified class names already assigned, for collision avoidance. */ - private final Set takenNames = new HashSet<>(); - - /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ - private final ClassLoader parentClassLoader; - - /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ - @Nullable - private final File additionalClassPathDir; - - /** For test use only: specifying a non-null list causes an annotation processing error. */ - private final List classNamesForAnnotationProcessing; - - // --- Factory methods --- + public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; + public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; /** * Creates a new InMemoryQueryCompiler. Uses the current thread's context classloader as the parent for per-batch @@ -138,7 +160,7 @@ public static InMemoryQueryCompiler create(@Nullable final File additionalClassP return new InMemoryQueryCompiler(additionalClassPathDir, null); } - static InMemoryQueryCompiler createForUnitTests() { + public static InMemoryQueryCompiler createForUnitTests() { return createForUnitTests(null); } @@ -146,6 +168,21 @@ static InMemoryQueryCompiler createForUnitTests(final List classNamesFor return new InMemoryQueryCompiler(null, classNamesForAnnotationProcessing); } + private final Map>> knownClasses = new HashMap<>(); + + /** Set of fully-qualified class names already assigned, for collision avoidance. */ + private final Set takenNames = new HashSet<>(); + + /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ + private final ClassLoader parentClassLoader; + + /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ + @Nullable + private final File additionalClassPathDir; + + // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. + private final List classNamesForAnnotationProcessing; + private InMemoryQueryCompiler( @Nullable final File additionalClassPathDir, final List classNamesForAnnotationProcessing) { @@ -153,9 +190,12 @@ private InMemoryQueryCompiler( this.additionalClassPathDir = additionalClassPathDir; this.parentClassLoader = Thread.currentThread().getContextClassLoader(); this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; - } - // --- Public API --- + if (log.isTraceEnabled()) { + log.trace().append("QueryCompiler Class Path: ").append(getClassPath()).append(File.pathSeparator) + .append(getJavaClassPath()).endl(); + } + } @Override public LogOutput append(LogOutput logOutput) { @@ -192,6 +232,22 @@ public void compile( throw new IllegalArgumentException("Requests and resolvers must be the same length"); } + final boolean shouldTrace = + log.isTraceEnabled() && Arrays.stream(requests).map(QueryCompilerRequest::packageNameRoot) + .anyMatch(InMemoryQueryCompiler::shouldTrace); + if (shouldTrace) { + log.trace().append("Compilation request for ").append((logOutput, queryCompilerRequests) -> { + logOutput.append("["); + for (int ii = 0; ii < queryCompilerRequests.length; ++ii) { + if (ii > 0) { + logOutput.append(", "); + } + requests[ii].appendSummary(logOutput); + } + logOutput.append("]"); + }, requests).endl(); + } + // noinspection unchecked final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; @@ -209,18 +265,21 @@ public void compile( newRequests.add(request); newResolvers.add(resolver); future = resolver.getFuture(); + } else if (shouldTrace) { + log.trace().append("Found existing future in knownClasses for ").append(request.className()) + .append(" (done=").append(future.isDone()).append(")").endl(); } allFutures[ii] = future; } } if (!newResolvers.isEmpty()) { - log.info().append("InMemoryQueryCompiler.compile: compiling ").append(newResolvers.size()) - .append(" new requests out of ").append(requests.length).append(" total").endl(); + // It's my job to fulfill these futures. try { compileHelper(newRequests, newResolvers); } catch (RuntimeException e) { - log.error().append("InMemoryQueryCompiler.compile: compileHelper threw ").append(e.toString()).endl(); + // These failures are not applicable to a single request, so we can't just complete the future and + // leave the failure in the cache. synchronized (this) { for (int ii = 0; ii < newRequests.size(); ++ii) { if (newResolvers.get(ii).completeExceptionally(e)) { @@ -230,25 +289,13 @@ public void compile( } throw e; } - log.info().append("InMemoryQueryCompiler.compile: compileHelper completed successfully").endl(); - } else { - log.info().append("InMemoryQueryCompiler.compile: all ").append(requests.length) - .append(" requests resolved from cache").endl(); } for (int ii = 0; ii < requests.length; ++ii) { try { final CompletionStageFuture> future = allFutures[ii]; - if (!future.isDone()) { - log.info().append("InMemoryQueryCompiler: waiting for future[").append(ii) - .append("] className=").append(requests[ii].className()) - .append(" classBody hash=").append( - Integer.toHexString(requests[ii].classBody().hashCode())) - .append(" future=").append(future.toString()) - .endl(); - } - final Class result = future.get(10, TimeUnit.SECONDS); - resolvers[ii].complete(result); + // TODO do we want a timeout here, knowing that this might leave other bad state? + resolvers[ii].complete(future.get(10, TimeUnit.SECONDS)); } catch (TimeoutException err) { final String msg = "InMemoryQueryCompiler: Timed out (10s) waiting for class compilation" + " request[" + ii + "] className=" + requests[ii].className() @@ -264,18 +311,16 @@ public void compile( } catch (ExecutionException err) { resolvers[ii].completeExceptionally(err.getCause()); } catch (InterruptedException err) { + // This can only occur if we are interrupted while waiting for the future to complete from another + // compilation request. Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); resolvers[ii].completeExceptionally(err); } catch (Throwable err) { resolvers[ii].completeExceptionally(err); } } - log.info().append("InMemoryQueryCompiler.compile: returning, all ").append(requests.length) - .append(" resolvers processed").endl(); } - // --- Compilation logic --- - private void compileHelper( @NotNull final List requests, @NotNull final List>> resolvers) { @@ -352,6 +397,7 @@ private void compileHelper( private void compileAndDefine(@NotNull final List requests) { final ExecutionContext executionContext = ExecutionContext.getContext(); final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); final int numTasks; @@ -366,6 +412,9 @@ private void compileAndDefine(@NotNull final List req jobScheduler = new OperationInitializerJobScheduler(); } + log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") + .append(numTasks).append(" tasks, ").append(requestsPerTask).endl(); + final JavaFileManager fileManager = acquireFileManager(); final AtomicReference exception = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(1); @@ -373,7 +422,7 @@ private void compileAndDefine(@NotNull final List req try { releaseFileManager(fileManager); } catch (Exception e) { - // ignore + // ignore errors here } finally { latch.countDown(); } @@ -400,9 +449,8 @@ private void compileAndDefine(@NotNull final List req onError); try { - log.info().append("InMemoryQueryCompiler.compileAndDefine: awaiting latch for ").append(numTasks) - .append(" task(s), ").append(requests.size()).append(" request(s), canParallelize=") - .append(canParallelize).endl(); + // TODO Probably should be configurable, probably shouldn't be indefinite? + // Allowing timeout like this may require cleanup final boolean completed = latch.await(10, TimeUnit.SECONDS); if (!completed) { final String msg = "InMemoryQueryCompiler.compileAndDefine: latch timed out after 10s!" @@ -410,7 +458,6 @@ private void compileAndDefine(@NotNull final List req log.error().append(msg).endl(); throw new UncheckedDeephavenException(msg); } - log.info().append("InMemoryQueryCompiler.compileAndDefine: latch completed").endl(); final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); if (perfEntry != null) { QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); @@ -456,6 +503,7 @@ private boolean doCompileAndDefineSingleRound( final String classPathAsString = getClassPath(); final List compilerOptions = Arrays.asList( "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail "--should-stop=ifError=GENERATE"); final MutableInt numFailures = new MutableInt(0); @@ -471,6 +519,7 @@ private boolean doCompileAndDefineSingleRound( final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); if (source == null) { + // If we have no source, then mark every request as a failure. final UncheckedDeephavenException err = new UncheckedDeephavenException( "Error Invoking Compiler, no source present in diagnostic:\n" + diagnostic.getMessage(Locale.getDefault())); @@ -481,6 +530,7 @@ private boolean doCompileAndDefineSingleRound( final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source numFailures.increment(); } }, @@ -493,7 +543,7 @@ private boolean doCompileAndDefineSingleRound( final String compilerOutputText = compilerOutput.toString(); if (!compilerOutputText.isEmpty()) { - log.info().append("Compiler output:\n").append(compilerOutputText).endl(); + log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); } if (!globalFailures.isEmpty()) { @@ -815,6 +865,7 @@ private static String getJavaClassPath() { final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); if (teamCityWorkDir != null) { + // We are running in TeamCity, get the classpath differently final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); if (classDirs != null) { for (File f : classDirs) { @@ -828,32 +879,43 @@ private static String getJavaClassPath() { javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); } } + + final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); + for (File f : jars) { + if (f.getName().endsWith(".jar")) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } } javaClasspath = javaClasspathBuilder.toString(); } + // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full + // class path. Look for this being used during compile time, so the full class path can be sent into the compile + // call. final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; if (javaClasspath.matches(intellijClassPathJarRegex)) { try { final Enumeration resources = - InMemoryQueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); - final java.util.jar.Attributes.Name createdByAttribute = - new java.util.jar.Attributes.Name("Created-By"); - final java.util.jar.Attributes.Name classPathAttribute = - new java.util.jar.Attributes.Name("Class-Path"); + QueryCompilerImpl.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); + final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); + final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); while (resources.hasMoreElements()) { - final java.util.jar.Manifest manifest = - new java.util.jar.Manifest(resources.nextElement().openStream()); - final java.util.jar.Attributes attributes = manifest.getMainAttributes(); + // Check all manifests -- looking for the Intellij created one + final Manifest manifest = new Manifest(resources.nextElement().openStream()); + final Attributes attributes = manifest.getMainAttributes(); final Object createdBy = attributes.get(createdByAttribute); if ("IntelliJ IDEA".equals(createdBy)) { final String extendedClassPath = (String) attributes.get(classPathAttribute); if (extendedClassPath != null) { + // Parses the files in the manifest description an changes their format to drop the "file:/" + // and use the default path separator final String filePaths = Stream.of(extendedClassPath.split("file:/")) .map(String::trim) .filter(fileName -> !fileName.isEmpty()) .collect(Collectors.joining(File.pathSeparator)); + // Remove the classpath jar in question, and expand it with the files from the manifest javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) .collect(Collectors.joining(File.pathSeparator)); diff --git a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java index 99077f3a4e6..e5d81e2a430 100644 --- a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java +++ b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java @@ -114,7 +114,7 @@ public void run() { } private QueryCompiler compilerForUnitTests() { - return InMemoryQueryCompiler.create(null); + return InMemoryQueryCompiler.createForUnitTests(); } @Test From efefb125998de4206a1a8b42d291f585ff5cd5e3 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 25 Jun 2026 10:47:17 -0500 Subject: [PATCH 03/12] remove static call --- .../io/deephaven/engine/util/GroovyDeephavenSession.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index d163ff6ee74..3a4f8926154 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -690,12 +690,6 @@ && isAnInteger(aClass.getName().substring(SCRIPT_PREFIX.length()))) { notifiedQueryLibrary = true; executionContext.getQueryLibrary().updateVersionString(); } - - try { - QueryCompilerImpl.writeClass(classCacheDirectory, entry.getKey(), entry.getValue()); - } catch (IOException e) { - throw new RuntimeException(e); - } } } } From dfbfc980b9c1ef0528a718f8693792734b9beec6 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 25 Jun 2026 10:56:42 -0500 Subject: [PATCH 04/12] Remove old class and rename new to take its place --- docs/groovy/conceptual/execution-context.md | 6 +- docs/python/conceptual/execution-context.md | 8 +- .../engine/context/TestQueryCompiler.java | 6 +- .../engine/context/InMemoryQueryCompiler.java | 933 ------------ .../engine/context/QueryCompilerImpl.java | 1274 ++++++----------- .../table/impl/select/ConditionFilter.java | 4 +- .../table/impl/select/DhFormulaColumn.java | 6 +- .../impl/select/QueryScopeParamTypeUtil.java | 4 +- .../select/codegen/JavaKernelBuilder.java | 6 +- .../engine/util/AbstractScriptSession.java | 2 +- .../engine/util/DynamicCompileUtils.java | 6 +- .../engine/util/GroovyDeephavenSession.java | 2 +- .../impl/TestEventDrivenUpdateGraph.java | 6 +- .../scripts/TestGroovyDeephavenSession.java | 4 +- .../engine/context/TestExecutionContext.java | 2 +- .../testcase/RefreshingTableTestCase.java | 6 +- .../parquet/table/TableWriteBenchmark.java | 4 +- .../impl/util/TestUpdateAncestorViz.java | 4 +- 18 files changed, 473 insertions(+), 1810 deletions(-) delete mode 100644 engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java diff --git a/docs/groovy/conceptual/execution-context.md b/docs/groovy/conceptual/execution-context.md index e38d18f5e36..c658579bc7d 100644 --- a/docs/groovy/conceptual/execution-context.md +++ b/docs/groovy/conceptual/execution-context.md @@ -225,7 +225,7 @@ The following example demonstrates how to build an `ExecutionContext` from scrat ```groovy skip-test import io.deephaven.engine.context.ExecutionContext -import io.deephaven.engine.context.InMemoryQueryCompiler +import io.deephaven.engine.context.QueryCompilerImpl import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph import io.deephaven.engine.updategraph.OperationInitializer import java.nio.file.Files @@ -235,7 +235,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler(InMemoryQueryCompiler.create(Files.createTempDirectory("qc_").toFile())) + .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile())) .build() ``` @@ -262,7 +262,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(eventDrivenGraph) - .setQueryCompiler(InMemoryQueryCompiler.create(Files.createTempDirectory("qc_").toFile())) + .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile())) .build() ``` diff --git a/docs/python/conceptual/execution-context.md b/docs/python/conceptual/execution-context.md index 88a27aa5ebe..23fbbdf6edd 100644 --- a/docs/python/conceptual/execution-context.md +++ b/docs/python/conceptual/execution-context.md @@ -244,7 +244,7 @@ from deephaven.jcompat import j_hashmap import tempfile ExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") -InMemoryQueryCompiler = jpy.get_type("io.deephaven.engine.context.InMemoryQueryCompiler") +QueryCompilerImpl = jpy.get_type("io.deephaven.engine.context.QueryCompilerImpl") PeriodicUpdateGraph = jpy.get_type( "io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph" ) @@ -261,7 +261,7 @@ execution_context = ( .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) .setQueryCompiler( - InMemoryQueryCompiler.create( + QueryCompilerImpl.create( jpy.get_type("java.io.File")(temp_dir), ) ) @@ -287,7 +287,7 @@ import jpy import tempfile ExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") -InMemoryQueryCompiler = jpy.get_type("io.deephaven.engine.context.InMemoryQueryCompiler") +QueryCompilerImpl = jpy.get_type("io.deephaven.engine.context.QueryCompilerImpl") EventDrivenUpdateGraph = jpy.get_type( "io.deephaven.engine.updategraph.impl.EventDrivenUpdateGraph" ) @@ -306,7 +306,7 @@ execution_context = ( .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(event_driven_graph) .setQueryCompiler( - InMemoryQueryCompiler.create( + QueryCompilerImpl.create( jpy.get_type("java.io.File")(temp_dir), ) ) diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index ca0a53170c0..fd5c1ff5ba1 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -65,7 +65,7 @@ public void setUp() { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(InMemoryQueryCompiler.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) .build() .open(); } @@ -369,8 +369,8 @@ public void testBadCompile() { CompletionStageFuture.make(), }; - final InMemoryQueryCompiler badCompiler = - InMemoryQueryCompiler.createForUnitTests(List.of("InvalidClassArgument")); + final QueryCompilerImpl badCompiler = + QueryCompilerImpl.createForUnitTests(List.of("InvalidClassArgument")); UncheckedDeephavenException e = org.junit.Assert.assertThrows(UncheckedDeephavenException.class, () -> badCompiler.compile(requests, resolvers)); org.junit.Assert.assertEquals("Error Invoking Compiler, no source present in diagnostic:\n" + diff --git a/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java b/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java deleted file mode 100644 index fe9a1728762..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java +++ /dev/null @@ -1,933 +0,0 @@ -// -// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending -// -package io.deephaven.engine.context; - -import io.deephaven.UncheckedDeephavenException; -import io.deephaven.base.FileUtils; -import io.deephaven.base.log.LogOutput; -import io.deephaven.base.log.LogOutputAppendable; -import io.deephaven.base.verify.Assert; -import io.deephaven.configuration.Configuration; -import io.deephaven.engine.context.util.SynchronizedJavaFileManager; -import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; -import io.deephaven.engine.table.impl.util.ImmediateJobScheduler; -import io.deephaven.engine.table.impl.util.JobScheduler; -import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.log.impl.LogOutputStringImpl; -import io.deephaven.io.logger.Logger; -import io.deephaven.util.ByteUtils; -import io.deephaven.util.CompletionStageFuture; -import io.deephaven.util.mutable.MutableInt; -import org.apache.commons.text.StringEscapeUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import javax.tools.*; -import java.io.*; -import java.lang.reflect.Field; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.jar.Attributes; -import java.util.jar.Manifest; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory and defines the resulting - * classes via {@link ClassLoader#defineClass}, avoiding filesystem writes for compilation output. - * - *

- * The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory - * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the - * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Class nor this - * compiler instance are reachable. - * - *

Constraints

- *
    - *
  • Compiled classes must not depend on other classes compiled by this or any other QueryCompiler instance.
    • - *
  • The caller must ensure the parent classloader already contains any classes referenced by compiled formulas (e.g., - * Groovy-defined classes).
    • - *
  • Each compilation request produces a single top-level class (matching {@link QueryCompilerRequest#className()}) - * which may contain static inner classes and anonymous classes.
    • - *
- */ -public class InMemoryQueryCompiler implements QueryCompiler, LogOutputAppendable { - - private static final Logger log = LoggerFactory.getLogger(InMemoryQueryCompiler.class); - /** - * We pick a number just shy of 65536, leaving a little elbow room for good luck. - */ - private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; - - private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); - - private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; - - private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); - - private static final String TRACE_PREFIX_PROPERTY = "QueryCompiler.tracePrefixes"; - private static final String TRACE_EXCLUDE_PREFIX_PROPERTY = "QueryCompiler.excludeTracePrefixes"; - private static final Set TRACE_INCLUDE_PREFIXES = computeTracePackages(TRACE_PREFIX_PROPERTY); - private static final Set TRACE_EXCLUDE_PREFIXES = computeTracePackages(TRACE_EXCLUDE_PREFIX_PROPERTY); - - private static Set computeTracePackages(final String property) { - if (!Configuration.getInstance().hasProperty(property)) { - return Collections.emptySet(); - } - final String propertyValue = Configuration.getInstance().getProperty(property); - return Arrays.stream(propertyValue.split(",")).map(String::trim).collect(Collectors.toSet()); - } - - /** - * Should this class (or package) be traced? Even with only trace logging, we may not want to flood the log with - * "uninteresting" classes, so we provide an inclusion and an exclusion list. - * - *

- * Excludes take precedence over includes. - *

- * - * @param className the class/package name to check against our trace prefixes - * @return if this class/package should be traced - */ - private static boolean shouldTrace(String className) { - if (!log.isTraceEnabled()) { - return false; - } - if (TRACE_EXCLUDE_PREFIXES.stream().anyMatch(className::startsWith)) { - return false; - } - return TRACE_INCLUDE_PREFIXES.stream().anyMatch(className::startsWith); - } - - - private static JavaCompiler compiler; - private static final AtomicReference fileManagerCache = new AtomicReference<>(); - - private static void ensureJavaCompiler() { - synchronized (InMemoryQueryCompiler.class) { - if (compiler == null) { - compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException( - "No Java compiler provided - are you using a JRE instead of a JDK?"); - } - } - } - } - - private static JavaFileManager acquireFileManager() { - JavaFileManager fileManager = fileManagerCache.getAndSet(null); - if (fileManager == null) { - fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); - } - return fileManager; - } - - private static void releaseFileManager(@NotNull final JavaFileManager fileManager) { - // Reusing the file manager saves a lot of the time in the compilation process. However, we need to be careful - // to avoid keeping too many file handles open so we'll limit ourselves to just one outstanding file manager. - if (!fileManagerCache.compareAndSet(null, fileManager)) { - try { - fileManager.close(); - } catch (final IOException err) { - throw new UncheckedIOException("Could not close JavaFileManager", err); - } - } - } - - public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; - public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; - - /** - * Creates a new InMemoryQueryCompiler. Uses the current thread's context classloader as the parent for per-batch - * classloaders (as required by {@link QueryCompiler}'s contract). - * - * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) - */ - public static InMemoryQueryCompiler create(@Nullable final File additionalClassPathDir) { - return new InMemoryQueryCompiler(additionalClassPathDir, null); - } - - public static InMemoryQueryCompiler createForUnitTests() { - return createForUnitTests(null); - } - - static InMemoryQueryCompiler createForUnitTests(final List classNamesForAnnotationProcessing) { - return new InMemoryQueryCompiler(null, classNamesForAnnotationProcessing); - } - - private final Map>> knownClasses = new HashMap<>(); - - /** Set of fully-qualified class names already assigned, for collision avoidance. */ - private final Set takenNames = new HashSet<>(); - - /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ - private final ClassLoader parentClassLoader; - - /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ - @Nullable - private final File additionalClassPathDir; - - // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. - private final List classNamesForAnnotationProcessing; - - private InMemoryQueryCompiler( - @Nullable final File additionalClassPathDir, - final List classNamesForAnnotationProcessing) { - ensureJavaCompiler(); - this.additionalClassPathDir = additionalClassPathDir; - this.parentClassLoader = Thread.currentThread().getContextClassLoader(); - this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; - - if (log.isTraceEnabled()) { - log.trace().append("QueryCompiler Class Path: ").append(getClassPath()).append(File.pathSeparator) - .append(getJavaClassPath()).endl(); - } - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append("InMemoryQueryCompiler{additionalClassPathDir=") - .append(additionalClassPathDir == null ? "null" : additionalClassPathDir.getAbsolutePath()) - .append("}"); - } - - @Override - public String toString() { - return new LogOutputStringImpl().append(this).toString(); - } - - /** - * Enables or disables compilation logging. - * - * @param logEnabled Whether logging should be enabled - * @return The value of {@code logEnabled} before calling this method. - */ - public static boolean setLogEnabled(boolean logEnabled) { - boolean original = InMemoryQueryCompiler.logEnabled; - InMemoryQueryCompiler.logEnabled = logEnabled; - return original; - } - - @Override - public void compile( - @NotNull final QueryCompilerRequest[] requests, - @NotNull final CompletionStageFuture.Resolver>[] resolvers) { - if (requests.length == 0) { - return; - } - if (requests.length != resolvers.length) { - throw new IllegalArgumentException("Requests and resolvers must be the same length"); - } - - final boolean shouldTrace = - log.isTraceEnabled() && Arrays.stream(requests).map(QueryCompilerRequest::packageNameRoot) - .anyMatch(InMemoryQueryCompiler::shouldTrace); - if (shouldTrace) { - log.trace().append("Compilation request for ").append((logOutput, queryCompilerRequests) -> { - logOutput.append("["); - for (int ii = 0; ii < queryCompilerRequests.length; ++ii) { - if (ii > 0) { - logOutput.append(", "); - } - requests[ii].appendSummary(logOutput); - } - logOutput.append("]"); - }, requests).endl(); - } - - // noinspection unchecked - final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; - - final List newRequests = new ArrayList<>(); - final List>> newResolvers = new ArrayList<>(); - - synchronized (this) { - for (int ii = 0; ii < requests.length; ++ii) { - final QueryCompilerRequest request = requests[ii]; - final CompletionStageFuture.Resolver> resolver = resolvers[ii]; - - CompletionStageFuture> future = - knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); - if (future == null) { - newRequests.add(request); - newResolvers.add(resolver); - future = resolver.getFuture(); - } else if (shouldTrace) { - log.trace().append("Found existing future in knownClasses for ").append(request.className()) - .append(" (done=").append(future.isDone()).append(")").endl(); - } - allFutures[ii] = future; - } - } - - if (!newResolvers.isEmpty()) { - // It's my job to fulfill these futures. - try { - compileHelper(newRequests, newResolvers); - } catch (RuntimeException e) { - // These failures are not applicable to a single request, so we can't just complete the future and - // leave the failure in the cache. - synchronized (this) { - for (int ii = 0; ii < newRequests.size(); ++ii) { - if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newRequests.get(ii).classBody()); - } - } - } - throw e; - } - } - - for (int ii = 0; ii < requests.length; ++ii) { - try { - final CompletionStageFuture> future = allFutures[ii]; - // TODO do we want a timeout here, knowing that this might leave other bad state? - resolvers[ii].complete(future.get(10, TimeUnit.SECONDS)); - } catch (TimeoutException err) { - final String msg = "InMemoryQueryCompiler: Timed out (10s) waiting for class compilation" - + " request[" + ii + "] className=" + requests[ii].className() - + " future=" + allFutures[ii] - + " isDone=" + allFutures[ii].isDone(); - log.error().append(msg).endl(); - // Fail all remaining resolvers and throw immediately - final UncheckedDeephavenException timeout = new UncheckedDeephavenException(msg); - for (int jj = ii; jj < requests.length; ++jj) { - resolvers[jj].completeExceptionally(timeout); - } - throw timeout; - } catch (ExecutionException err) { - resolvers[ii].completeExceptionally(err.getCause()); - } catch (InterruptedException err) { - // This can only occur if we are interrupted while waiting for the future to complete from another - // compilation request. - Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); - resolvers[ii].completeExceptionally(err); - } catch (Throwable err) { - resolvers[ii].completeExceptionally(err); - } - } - } - - private void compileHelper( - @NotNull final List requests, - @NotNull final List>> resolvers) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } - - // Assign unique FQ class names for each request - final String[] fqClassNames = new String[requests.size()]; - final String[] packageNames = new String[requests.size()]; - - synchronized (this) { - for (int ii = 0; ii < requests.size(); ++ii) { - final QueryCompilerRequest request = requests.get(ii); - final String hashText = ByteUtils.byteArrToHex(digest.digest( - request.classBody().getBytes(StandardCharsets.UTF_8))); - - String fqClassName = null; - String packageName = null; - for (int pi = 0; pi < 128; ++pi) { - final String packageNameSuffix = "c_" + hashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - packageName = request.getPackageName(packageNameSuffix); - fqClassName = packageName + "." + request.className(); - if (!takenNames.contains(fqClassName)) { - break; - } - fqClassName = null; - } - if (fqClassName == null) { - resolvers.get(ii).completeExceptionally(new IllegalStateException( - "Unable to assign unique class name for " + request.className())); - continue; - } - takenNames.add(fqClassName); - fqClassNames[ii] = fqClassName; - packageNames[ii] = packageName; - } - } - - // Build compilation attempts for requests that got a valid name - final List attempts = new ArrayList<>(); - for (int ii = 0; ii < requests.size(); ++ii) { - if (fqClassNames[ii] == null) { - continue; // already failed - } - attempts.add(new CompilationRequestAttempt( - requests.get(ii), packageNames[ii], fqClassNames[ii], resolvers.get(ii))); - } - - if (attempts.isEmpty()) { - return; - } - - // Compile and define - compileAndDefine(attempts); - - // Validate _CLASS_BODY_ field on successfully defined classes - for (int ii = 0; ii < requests.size(); ++ii) { - final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); - if (resolver.getFuture().isDone()) { - continue; // already completed (success or failure) - } - // This shouldn't happen - compileAndDefine should have resolved everything - resolver.completeExceptionally(new IllegalStateException( - "Class was not resolved after compilation: " + fqClassNames[ii])); - } - } - - private void compileAndDefine(@NotNull final List requests) { - final ExecutionContext executionContext = ExecutionContext.getContext(); - final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); - - final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - - final int numTasks; - final JobScheduler jobScheduler; - - final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); - if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { - numTasks = 1; - jobScheduler = new ImmediateJobScheduler(); - } else { - numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; - jobScheduler = new OperationInitializerJobScheduler(); - } - - log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") - .append(numTasks).append(" tasks, ").append(requestsPerTask).endl(); - - final JavaFileManager fileManager = acquireFileManager(); - final AtomicReference exception = new AtomicReference<>(); - final CountDownLatch latch = new CountDownLatch(1); - final Runnable cleanup = () -> { - try { - releaseFileManager(fileManager); - } catch (Exception e) { - // ignore errors here - } finally { - latch.countDown(); - } - }; - - final Consumer onError = err -> { - if (err instanceof RuntimeException) { - exception.set((RuntimeException) err); - } else { - exception.set(new UncheckedDeephavenException("Error during compilation", err)); - } - cleanup.run(); - }; - - jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, - 0, numTasks, (context, jobId, nestedErrorConsumer) -> { - final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - doCompileAndDefine(fileManager, requests, startInclusive, endExclusive); - }, - () -> { - }, - cleanup, - onError); - - try { - // TODO Probably should be configurable, probably shouldn't be indefinite? - // Allowing timeout like this may require cleanup - final boolean completed = latch.await(10, TimeUnit.SECONDS); - if (!completed) { - final String msg = "InMemoryQueryCompiler.compileAndDefine: latch timed out after 10s!" - + " numTasks=" + numTasks + " requests=" + requests.size(); - log.error().append(msg).endl(); - throw new UncheckedDeephavenException(msg); - } - final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); - if (perfEntry != null) { - QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); - } - final RuntimeException err = exception.get(); - if (err != null) { - throw err; - } - } catch (final InterruptedException e) { - throw new CancellationException("interrupted while compiling"); - } - } - - private void doCompileAndDefine( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - final int startInclusive, - final int endExclusive) { - final List toRetry = new ArrayList<>(); - final boolean wantRetry = doCompileAndDefineSingleRound( - fileManager, requests, startInclusive, endExclusive, toRetry); - if (!wantRetry) { - return; - } - // Retry non-failing requests from the first pass - final List ignored = new ArrayList<>(); - if (doCompileAndDefineSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { - throw new IllegalStateException("Unexpected failure during second pass of compilation"); - } - } - - private boolean doCompileAndDefineSingleRound( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - final int startInclusive, - final int endExclusive, - @NotNull final List toRetry) { - - // Create an in-memory file manager that captures compiled output - final InMemoryOutputFileManager outputFm = new InMemoryOutputFileManager(fileManager); - final StringWriter compilerOutput = new StringWriter(); - - final String classPathAsString = getClassPath(); - final List compilerOptions = Arrays.asList( - "-cp", classPathAsString, - // this option allows the compiler to attempt to process all source files even if some of them fail - "--should-stop=ifError=GENERATE"); - - final MutableInt numFailures = new MutableInt(0); - final List globalFailures = new ArrayList<>(); - - compiler.getTask(compilerOutput, - outputFm, - diagnostic -> { - if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { - return; - } - - final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); - - if (source == null) { - // If we have no source, then mark every request as a failure. - final UncheckedDeephavenException err = new UncheckedDeephavenException( - "Error Invoking Compiler, no source present in diagnostic:\n" - + diagnostic.getMessage(Locale.getDefault())); - globalFailures.add(err); - return; - } - - final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " - + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); - if (source.resolver.completeExceptionally(err)) { - // only count the first failure for each source - numFailures.increment(); - } - }, - compilerOptions, - classNamesForAnnotationProcessing, - requests.subList(startInclusive, endExclusive).stream() - .map(CompilationRequestAttempt::makeSource) - .collect(Collectors.toList())) - .call(); - - final String compilerOutputText = compilerOutput.toString(); - if (!compilerOutputText.isEmpty()) { - log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); - } - - if (!globalFailures.isEmpty()) { - final RuntimeException e0 = globalFailures.get(0); - for (int ii = 1; ii < globalFailures.size(); ++ii) { - e0.addSuppressed(globalFailures.get(ii)); - } - throw e0; - } - - final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; - - // Define compiled classes into a per-batch classloader - final Map compiledClasses = outputFm.getCompiledClasses(); - log.info().append("InMemoryQueryCompiler: compilation produced ").append(compiledClasses.size()) - .append(" classes, numFailures=").append(numFailures.get()) - .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") - .endl(); - if (!compiledClasses.isEmpty()) { - // Use the current thread's context classloader as parent so that Groovy-defined classes - // (which may have been loaded after this InMemoryQueryCompiler was constructed) are visible. - final ClassLoader currentContextCl = Thread.currentThread().getContextClassLoader(); - final ClassLoader batchParent = currentContextCl != null ? currentContextCl : parentClassLoader; - final BatchClassLoader batchCl = new BatchClassLoader(batchParent, compiledClasses); - - for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { - if (request.resolver.getFuture().isDone()) { - // already failed - continue; - } - - if (!compiledClasses.containsKey(request.fqClassName)) { - if (wantRetry) { - toRetry.add(request); - } - continue; - } - - // Load the top-level class (which triggers loading of inner/anonymous classes as needed) - final Class clazz; - try { - clazz = batchCl.loadClass(request.fqClassName); - } catch (ClassNotFoundException e) { - request.resolver.completeExceptionally(new UncheckedDeephavenException( - "Failed to load compiled class: " + request.fqClassName, e)); - continue; - } - - // Validate the identifying field - final String identifyingFieldValue = loadIdentifyingField(clazz); - if (!request.request.classBody().equals(identifyingFieldValue)) { - request.resolver.completeExceptionally(new IllegalStateException( - "Compiled class body validation failed for " + request.fqClassName)); - continue; - } - - // Notify caller with codeLog if requested - request.request.codeLog().ifPresent( - sb -> sb.append(makeFinalCode( - request.request.className(), request.request.classBody(), request.packageName))); - - // Complete the future - log.info().append("InMemoryQueryCompiler: resolving ").append(request.fqClassName).endl(); - request.resolver.complete(clazz); - - // Canonicalize the knownClasses entry - synchronized (this) { - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, request.resolver.getFuture()); - } - } - } else { - // No output at all - if there are non-failed requests, they need retry - for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { - if (!request.resolver.getFuture().isDone() && wantRetry) { - toRetry.add(request); - } - } - } - - return wantRetry && !toRetry.isEmpty(); - } - - // --- Classpath construction --- - - private String getClassPath() { - final StringBuilder sb = new StringBuilder(getJavaClassPath()); - if (additionalClassPathDir != null) { - sb.append(File.pathSeparator).append(additionalClassPathDir.getAbsolutePath()); - } - return sb.toString(); - } - - // --- BatchClassLoader --- - - /** - * A classloader that defines classes from a map of name→bytes. All classes from a single compilation batch are - * loaded together. The parent classloader handles all other class resolution. - */ - private static class BatchClassLoader extends ClassLoader { - private final Map classBytes; - - BatchClassLoader(@NotNull ClassLoader parent, @NotNull Map classBytes) { - super(parent); - this.classBytes = classBytes; - } - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - final byte[] bytes = classBytes.get(name); - if (bytes != null) { - return defineClass(name, bytes, 0, bytes.length); - } - throw new ClassNotFoundException(name); - } - } - - // --- InMemoryOutputFileManager --- - - /** - * A forwarding JavaFileManager that intercepts class output, storing compiled bytecode in memory rather than - * writing to the filesystem. All other operations are delegated. - */ - private static class InMemoryOutputFileManager extends ForwardingJavaFileManager { - private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); - - InMemoryOutputFileManager(JavaFileManager delegate) { - super(delegate); - } - - Map getCompiledClasses() { - final Map result = new HashMap<>(); - for (Map.Entry entry : outputClasses.entrySet()) { - result.put(entry.getKey(), entry.getValue().getBytes()); - } - return result; - } - - @Override - public JavaFileObject getJavaFileForOutput( - Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { - final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); - outputClasses.put(className, fileObject); - return fileObject; - } - - @Override - public boolean hasLocation(Location location) { - if (location == StandardLocation.CLASS_OUTPUT) { - return true; - } - return super.hasLocation(location); - } - } - - // --- InMemoryClassFileObject --- - - private static class InMemoryClassFileObject extends SimpleJavaFileObject { - private final String className; - private ByteArrayOutputStream outputStream; - - InMemoryClassFileObject(String className) { - super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); - this.className = className; - } - - @Override - public OutputStream openOutputStream() { - outputStream = new ByteArrayOutputStream(); - return outputStream; - } - - byte[] getBytes() { - if (outputStream == null) { - throw new IllegalStateException("No bytes available for " + className); - } - return outputStream.toByteArray(); - } - } - - // --- Source representation --- - - private static class JavaSourceFromString extends SimpleJavaFileObject { - final String description; - final String code; - final CompletionStageFuture.Resolver> resolver; - - JavaSourceFromString( - final String description, - final String name, - final String code, - final CompletionStageFuture.Resolver> resolver) { - super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); - this.description = description; - this.code = code; - this.resolver = resolver; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private static class CompilationRequestAttempt { - final String description; - final String fqClassName; - final String finalCode; - final String packageName; - final QueryCompilerRequest request; - final CompletionStageFuture.Resolver> resolver; - - private CompilationRequestAttempt( - @NotNull final QueryCompilerRequest request, - @NotNull final String packageName, - @NotNull final String fqClassName, - @NotNull final CompletionStageFuture.Resolver> resolver) { - this.description = request.description(); - this.fqClassName = fqClassName; - this.resolver = resolver; - this.packageName = packageName; - this.request = request; - - finalCode = makeFinalCode(request.className(), request.classBody(), packageName); - - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); - } - } - - JavaSourceFromString makeSource() { - return new JavaSourceFromString(description, fqClassName, finalCode, resolver); - } - } - - // --- Utilities --- - - private static String loadIdentifyingField(Class c) { - try { - final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); - return (String) field.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Malformed class in cache", e); - } - } - - static String makeFinalCode(String className, String classBody, String packageName) { - if (classBody.contains("$CLASSNAME$")) { - throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" - + " the final class name affects the compiled byte code and therefore cannot be dynamically " - + "replaced."); - } - - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; - } - - /** - * Transform a string into the corresponding Java source code that compiles into that string. - */ - public static String createEscapedJoinedString(final String originalString) { - return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); - } - - public static String createEscapedJoinedString(final String originalString, int maxStringLength) { - final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - - for (int ii = 0; ii < splits.length; ++ii) { - final String escaped = StringEscapeUtils.escapeJava(splits[ii]); - splits[ii] = "\"" + escaped + "\""; - } - assert splits.length > 0; - if (splits.length == 1) { - return splits[0]; - } - final String formattedInnards = String.join(",\n", splits); - return "String.join(\"\", " + formattedInnards + ")"; - } - - private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { - final List splits = new ArrayList<>(); - int previousEnd = 0; - int currentByteCount = 0; - for (int ii = 0; ii < originalString.length(); ++ii) { - final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); - if (currentByteCount + bytesConsumed > maxBytes) { - splits.add(originalString.substring(previousEnd, ii)); - previousEnd = ii; - currentByteCount = 0; - } - currentByteCount += bytesConsumed; - } - splits.add(originalString.substring(previousEnd)); - return splits.toArray(String[]::new); - } - - private static int calcBytesConsumed(final char ch) { - if (ch == 0) { - return 2; - } - if (ch <= 0x7f) { - return 1; - } - if (ch <= 0x7ff) { - return 2; - } - return 3; - } - - /** - * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables - */ - private static String getJavaClassPath() { - String javaClasspath; - { - final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); - - final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); - if (teamCityWorkDir != null) { - // We are running in TeamCity, get the classpath differently - final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - if (classDirs != null) { - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - if (testDirs != null) { - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - - final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); - for (File f : jars) { - if (f.getName().endsWith(".jar")) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - } - javaClasspath = javaClasspathBuilder.toString(); - } - - // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full - // class path. Look for this being used during compile time, so the full class path can be sent into the compile - // call. - final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; - if (javaClasspath.matches(intellijClassPathJarRegex)) { - try { - final Enumeration resources = - QueryCompilerImpl.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); - final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); - final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); - while (resources.hasMoreElements()) { - // Check all manifests -- looking for the Intellij created one - final Manifest manifest = new Manifest(resources.nextElement().openStream()); - final Attributes attributes = manifest.getMainAttributes(); - final Object createdBy = attributes.get(createdByAttribute); - if ("IntelliJ IDEA".equals(createdBy)) { - final String extendedClassPath = (String) attributes.get(classPathAttribute); - if (extendedClassPath != null) { - // Parses the files in the manifest description an changes their format to drop the "file:/" - // and use the default path separator - final String filePaths = Stream.of(extendedClassPath.split("file:/")) - .map(String::trim) - .filter(fileName -> !fileName.isEmpty()) - .collect(Collectors.joining(File.pathSeparator)); - - // Remove the classpath jar in question, and expand it with the files from the manifest - javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) - .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) - .collect(Collectors.joining(File.pathSeparator)); - } - } - } - } catch (IOException e) { - throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); - } - } - return javaClasspath; - } -} - - diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 6cce3289372..24e330977a5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -9,7 +9,6 @@ import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; -import io.deephaven.configuration.DataDir; import io.deephaven.engine.context.util.SynchronizedJavaFileManager; import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; @@ -17,7 +16,6 @@ import io.deephaven.engine.table.impl.util.JobScheduler; import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.log.LogEntry; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; @@ -25,31 +23,44 @@ import io.deephaven.util.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import javax.tools.*; import java.io.*; import java.lang.reflect.Field; -import java.net.MalformedURLException; import java.net.URI; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory and defines the resulting + * classes via {@link ClassLoader#defineClass}, avoiding filesystem writes for compilation output. + * + *

+ * The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory + * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the + * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Class nor this + * compiler instance are reachable. + * + *

Constraints

+ *
    + *
  • Compiled classes must not depend on other classes compiled by this or any other QueryCompiler instance.
    • + *
  • The caller must ensure the parent classloader already contains any classes referenced by compiled formulas (e.g., + * Groovy-defined classes).
    • + *
  • Each compilation request produces a single top-level class (matching {@link QueryCompilerRequest#className()}) + * which may contain static inner classes and anonymous classes.
    • + *
+ */ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { private static final Logger log = LoggerFactory.getLogger(QueryCompilerImpl.class); @@ -59,19 +70,9 @@ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); - private static final int MAX_CLASS_COLLISIONS = 128; private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; - private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; - private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds - private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; - private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long CODEGEN_TIMEOUT_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long CODEGEN_LOOP_DELAY_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); - private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); private static final String TRACE_PREFIX_PROPERTY = "QueryCompiler.tracePrefixes"; @@ -147,56 +148,45 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; - public static QueryCompilerImpl create(File cacheDirectory, ClassLoader classLoader) { - return new QueryCompilerImpl(cacheDirectory, classLoader, true, null); + /** + * Creates a new QueryCompilerImpl. Uses the current thread's context classloader as the parent for per-batch + * classloaders (as required by {@link QueryCompiler}'s contract). + * + * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) + */ + public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir) { + return new QueryCompilerImpl(additionalClassPathDir, null); } - static QueryCompilerImpl createForUnitTests() { + public static QueryCompilerImpl createForUnitTests() { return createForUnitTests(null); } static QueryCompilerImpl createForUnitTests(final List classNamesForAnnotationProcessing) { - final Path queryCompilerDir = DataDir.get() - .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); - return new QueryCompilerImpl(queryCompilerDir.toFile(), QueryCompilerImpl.class.getClassLoader(), false, - classNamesForAnnotationProcessing); + return new QueryCompilerImpl(null, classNamesForAnnotationProcessing); } private final Map>> knownClasses = new HashMap<>(); - private final String[] dynamicPatterns = new String[] {DYNAMIC_CLASS_PREFIX, FORMULA_CLASS_PREFIX}; + /** Set of fully-qualified class names already assigned, for collision avoidance. */ + private final Set takenNames = new HashSet<>(); + + /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ + private final ClassLoader parentClassLoader; + + /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ + @Nullable + private final File additionalClassPathDir; - private final File classDestination; - private final Set additionalClassLocations; - private final WritableURLClassLoader ucl; // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. private final List classNamesForAnnotationProcessing; private QueryCompilerImpl( - @NotNull final File classDestination, - @NotNull final ClassLoader parentClassLoader, - boolean classDestinationIsAlsoClassSource, + @Nullable final File additionalClassPathDir, final List classNamesForAnnotationProcessing) { ensureJavaCompiler(); - - this.classDestination = classDestination; - ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + - classDestination.getAbsolutePath()); - additionalClassLocations = new LinkedHashSet<>(); - - URL[] urls = new URL[1]; - try { - urls[0] = (classDestination.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - this.ucl = new WritableURLClassLoader(urls, parentClassLoader); - log.trace().append("Class destination is ").append(classDestination.toString()).endl(); - - if (classDestinationIsAlsoClassSource) { - addClassSource(classDestination); - } - + this.additionalClassPathDir = additionalClassPathDir; + this.parentClassLoader = Thread.currentThread().getContextClassLoader(); this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; if (log.isTraceEnabled()) { @@ -207,7 +197,8 @@ private QueryCompilerImpl( @Override public LogOutput append(LogOutput logOutput) { - return logOutput.append("QueryCompiler{classDestination=").append(classDestination.getAbsolutePath()) + return logOutput.append("QueryCompilerImpl{additionalClassPathDir=") + .append(additionalClassPathDir == null ? "null" : additionalClassPathDir.getAbsolutePath()) .append("}"); } @@ -228,78 +219,6 @@ public static boolean setLogEnabled(boolean logEnabled) { return original; } - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data) - throws IOException { - writeClass(destinationDirectory, className, data, null); - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data, - final String message) throws IOException { - final File destinationFile = new File(destinationDirectory, - className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - - final boolean shouldTrace = shouldTrace(className); - - if (destinationFile.exists()) { - if (shouldTrace) { - log.trace().append("Destination class file already exists ").append(destinationFile.toString()).endl(); - } - - final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); - if (Arrays.equals(existingBytes, data)) { - if (message == null) { - log.info().append("Ignoring pushed class ").append(className) - .append(" because it already exists in this context!").endl(); - } else { - log.info().append("Ignoring pushed class ").append(className).append(message) - .append(" because it already exists in this context!").endl(); - } - return; - } else { - if (message == null) { - log.info().append("Pushed class ").append(className) - .append(" already exists in this context, but has changed!").endl(); - } else { - log.info().append("Pushed class ").append(className).append(message) - .append(" already exists in this context, but has changed!").endl(); - } - if (!destinationFile.delete()) { - throw new IOException("Could not delete existing class file: " + destinationFile); - } - } - } else if (shouldTrace) { - log.trace().append("Destination class file does not already exist ").append(destinationFile.toString()) - .endl(); - } - - final File parentDir = destinationFile.getParentFile(); - ensureDirectories(parentDir, - () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); - if (!destinationFile.createNewFile()) { - throw new UncheckedDeephavenException( - "Unable to create destination file " + destinationFile.getAbsolutePath()); - } - final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); - byteOutStream.write(data, 0, data.length); - final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); - byteOutStream.writeTo(fileOutStream); - fileOutStream.close(); - - if (shouldTrace) { - log.trace().append("Wrote destination class file ").append(destinationFile.toString()).endl(); - } - } - @Override public void compile( @NotNull final QueryCompilerRequest[] requests, @@ -372,7 +291,21 @@ public void compile( for (int ii = 0; ii < requests.length; ++ii) { try { - resolvers[ii].complete(allFutures[ii].get()); + final CompletionStageFuture> future = allFutures[ii]; + // TODO do we want a timeout here, knowing that this might leave other bad state? + resolvers[ii].complete(future.get(10, TimeUnit.SECONDS)); + } catch (TimeoutException err) { + final String msg = "Timed out (10s) waiting for class compilation" + + " request[" + ii + "] className=" + requests[ii].className() + + " future=" + allFutures[ii] + + " isDone=" + allFutures[ii].isDone(); + log.error().append(msg).endl(); + // Fail all remaining resolvers and throw immediately + final UncheckedDeephavenException timeout = new UncheckedDeephavenException(msg); + for (int jj = ii; jj < requests.length; ++jj) { + resolvers[jj].completeExceptionally(timeout); + } + throw timeout; } catch (ExecutionException err) { resolvers[ii].completeExceptionally(err.getCause()); } catch (InterruptedException err) { @@ -386,563 +319,411 @@ public void compile( } } - private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { - // File.mkdirs() checks for existence on entry, in which case it returns false. - // It may also return false on a failure to create. - // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could - // lose the race - // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() - // would return true) - if (!file.mkdirs() && !file.isDirectory()) { - throw new UncheckedDeephavenException(runtimeErrMsg.get()); + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); } - } - - private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { - return new URLClassLoader(ucl.getURLs(), ucl) { - // Once we find a class that is missing, we should not attempt to load it again, - // otherwise we can end up with a StackOverflow Exception - final HashSet missingClasses = new HashSet<>(); - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - final boolean shouldTrace = shouldTrace(name); - - // If we have a parameter that uses this class, return it - final Class paramClass = parameterClasses.get(name); - if (paramClass != null) { - if (shouldTrace) { - log.trace().append("findClass(").append(name).append("): matched parameter class").endl(); - } - return paramClass; - } - - // Unless we are looking for a formula or Groovy class, we should use the default behavior - if (!isFormulaClass(name)) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): not a formula class, delegating to super.findClass").endl(); - } - return super.findClass(name); - } - // if it is a groovy class, always try to use the instance in the shell - if (name.startsWith(DYNAMIC_CLASS_PREFIX)) { - try { - final Class dynamicResult = ucl.getParent().loadClass(name); - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): loaded dynamic class from parent of WritableURLClassLoader").endl(); - } - return dynamicResult; - } catch (final ClassNotFoundException ignored) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): dynamic class not found in parent of WritableURLClassLoader," - + " falling through") - .endl(); - } - // we'll try to load it otherwise - } - } + // Assign unique FQ class names for each request + final String[] fqClassNames = new String[requests.size()]; + final String[] packageNames = new String[requests.size()]; - // We've already not found this class, so we should not try to search again - if (missingClasses.contains(name)) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): previously cached as missing, delegating to super.findClass").endl(); - } - return super.findClass(name); - } + synchronized (this) { + for (int ii = 0; ii < requests.size(); ++ii) { + final QueryCompilerRequest request = requests.get(ii); + final String hashText = ByteUtils.byteArrToHex(digest.digest( + request.classBody().getBytes(StandardCharsets.UTF_8))); - final byte[] bytes; - try { - bytes = loadClassData(name); - } catch (IOException ioe) { - if (shouldTrace) { - log.trace().append("findClass(").append(name) - .append("): loadClassData failed (").append(ioe.getMessage()) - .append("), caching as missing and delegating to super.loadClass").endl(); + String fqClassName = null; + String packageName = null; + for (int pi = 0; pi < 128; ++pi) { + final String packageNameSuffix = "c_" + hashText + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + packageName = request.getPackageName(packageNameSuffix); + fqClassName = packageName + "." + request.className(); + if (!takenNames.contains(fqClassName)) { + break; } - missingClasses.add(name); - return super.loadClass(name); + fqClassName = null; } - if (shouldTrace) { - log.trace().append("findClass(").append(name).append("): defining class from ") - .append(bytes.length).append(" bytes").endl(); + if (fqClassName == null) { + resolvers.get(ii).completeExceptionally(new IllegalStateException( + "Unable to assign unique class name for " + request.className())); + continue; } - return defineClass(name, bytes, 0, bytes.length); - } - - @SuppressWarnings("BooleanMethodIsAlwaysInverted") - private boolean isFormulaClass(String name) { - return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); + takenNames.add(fqClassName); + fqClassNames[ii] = fqClassName; + packageNames[ii] = packageName; } + } - @Override - public Class loadClass(String name) throws ClassNotFoundException { - if (!isFormulaClass(name)) { - return super.loadClass(name); - } - return findClass(name); + // Build compilation attempts for requests that got a valid name + final List attempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + if (fqClassNames[ii] == null) { + continue; // already failed } + attempts.add(new CompilationRequestAttempt( + requests.get(ii), packageNames[ii], fqClassNames[ii], resolvers.get(ii))); + } - private byte[] loadClassData(String name) throws IOException { - final boolean shouldTrace = shouldTrace(name); - - final File destFile = new File(classDestination, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (destFile.exists()) { - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): found at ") - .append(destFile.toString()).endl(); - } - return Files.readAllBytes(destFile.toPath()); - } - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): not present at ") - .append(destFile.toString()).endl(); - } + if (attempts.isEmpty()) { + return; + } - for (File location : additionalClassLocations) { - final File checkFile = new File(location, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (checkFile.exists()) { - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): found at ") - .append(checkFile.toString()).endl(); - } - return Files.readAllBytes(checkFile.toPath()); - } - if (shouldTrace) { - log.trace().append("loadClassData(").append(name).append("): not present at ") - .append(checkFile.toString()).endl(); - } - } + // Compile and define + compileAndDefine(attempts); - if (shouldTrace) { - log.trace().append("loadClassData(").append(name) - .append("): not found in any class location").endl(); - } - throw new FileNotFoundException(name); + // Validate _CLASS_BODY_ field on successfully defined classes + for (int ii = 0; ii < requests.size(); ++ii) { + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + continue; // already completed (success or failure) } - }; + // This shouldn't happen - compileAndDefine should have resolved everything + resolver.completeExceptionally(new IllegalStateException( + "Class was not resolved after compilation: " + fqClassNames[ii])); + } } - private static class WritableURLClassLoader extends URLClassLoader { - private WritableURLClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); + private void compileAndDefine(@NotNull final List requests) { + final ExecutionContext executionContext = ExecutionContext.getContext(); + final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); + + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); + + final int numTasks; + final JobScheduler jobScheduler; + + final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); + if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { + numTasks = 1; + jobScheduler = new ImmediateJobScheduler(); + } else { + numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + jobScheduler = new OperationInitializerJobScheduler(); } - @Override - protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - final boolean shouldTrace = shouldTrace(name); - final int loaderId = System.identityHashCode(this); - - Class clazz = findLoadedClass(name); - if (clazz != null) { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") returned from findLoadedClass").endl(); - } - return clazz; - } + log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") + .append(numTasks).append(" tasks, ").append(requestsPerTask).endl(); + final JavaFileManager fileManager = acquireFileManager(); + final AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + final Runnable cleanup = () -> { try { - clazz = findClass(name); - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") returned from findClass").endl(); - } - } catch (ClassNotFoundException e) { - if (getParent() != null) { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") delegating to parent").endl(); - } - clazz = getParent().loadClass(name); - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name).append(") parent returned class").endl(); - } - } else { - if (shouldTrace) { - log.trace().append("WritableURLClassLoader@").append(loaderId) - .append(".loadClass(").append(name) - .append(") not found by findClass and no parent loader; returning null").endl(); - } - } + releaseFileManager(fileManager); + } catch (Exception e) { + // ignore errors here + } finally { + latch.countDown(); } + }; - if (resolve) { - resolveClass(clazz); + final Consumer onError = err -> { + if (err instanceof RuntimeException) { + exception.set((RuntimeException) err); + } else { + exception.set(new UncheckedDeephavenException("Error during compilation", err)); } - return clazz; - } + cleanup.run(); + }; - @Override - public synchronized void addURL(URL url) { - super.addURL(url); - } - } + jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, + 0, numTasks, (context, jobId, nestedErrorConsumer) -> { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + doCompileAndDefine(fileManager, requests, startInclusive, endExclusive); + }, + () -> { + }, + cleanup, + onError); - private void addClassSource(File classSourceDirectory) { - synchronized (additionalClassLocations) { - if (additionalClassLocations.contains(classSourceDirectory)) { - return; - } - additionalClassLocations.add(classSourceDirectory); - } try { - ucl.addURL(classSourceDirectory.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - } - - private File getClassDestination() { - return classDestination; - } - - private String getClassPath() { - StringBuilder sb = new StringBuilder(); - sb.append(classDestination.getAbsolutePath()); - synchronized (additionalClassLocations) { - for (File classLoc : additionalClassLocations) { - sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); + // TODO Probably should be configurable, probably shouldn't be indefinite? + // Allowing timeout like this may require cleanup + final boolean completed = latch.await(10, TimeUnit.SECONDS); + if (!completed) { + final String msg = "QueryCompilerImpl.compileAndDefine: latch timed out after 10s!" + + " numTasks=" + numTasks + " requests=" + requests.size(); + log.error().append(msg).endl(); + throw new UncheckedDeephavenException(msg); + } + final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); + if (perfEntry != null) { + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); + } + final RuntimeException err = exception.get(); + if (err != null) { + throw err; } + } catch (final InterruptedException e) { + throw new CancellationException("interrupted while compiling"); } - return sb.toString(); - } - - private static class CompilationState { - int nextProbeIndex; - boolean complete; - String packageName; - String fqClassName; } - private void compileHelper( - @NotNull final List requests, - @NotNull final List>> resolvers) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); + private void doCompileAndDefine( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + final boolean wantRetry = doCompileAndDefineSingleRound( + fileManager, requests, startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; } - - final String[] basicHashText = new String[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( - requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + // Retry non-failing requests from the first pass + final List ignored = new ArrayList<>(); + if (doCompileAndDefineSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { + throw new IllegalStateException("Unexpected failure during second pass of compilation"); } + } - int numComplete = 0; - final CompilationState[] states = new CompilationState[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - states[ii] = new CompilationState(); - } + private boolean doCompileAndDefineSingleRound( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + final int startInclusive, + final int endExclusive, + @NotNull final List toRetry) { - /* - * @formatter:off - * 1. try to resolve without compiling; retain next hash to try - * 2. compile all remaining with a single compilation task - * 3. goto step 1 if any are unresolved - * @formatter:on - */ + // Create an in-memory file manager that captures compiled output + final InMemoryOutputFileManager outputFm = new InMemoryOutputFileManager(fileManager); + final StringWriter compilerOutput = new StringWriter(); - while (numComplete < requests.size()) { - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { - continue; - } + final String classPathAsString = getClassPath(); + final List compilerOptions = Arrays.asList( + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); - next_probe: while (true) { - final int pi = state.nextProbeIndex++; - final String packageNameSuffix = "c_" + basicHashText[ii] - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; + final MutableInt numFailures = new MutableInt(0); + final List globalFailures = new ArrayList<>(); - final QueryCompilerRequest request = requests.get(ii); - if (pi >= MAX_CLASS_COLLISIONS) { - Exception err = new IllegalStateException("Found too many collisions for package name root " - + request.packageNameRoot() + ", class name=" + request.className() + ", class body " - + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); - resolvers.get(ii).completeExceptionally(err); - state.complete = true; - ++numComplete; - break; + compiler.getTask(compilerOutput, + outputFm, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; } - state.packageName = request.getPackageName(packageNameSuffix); - state.fqClassName = state.packageName + "." + request.className(); - - for (int jj = 0; jj < ii; ++jj) { - if (states[jj].fqClassName.equals(state.fqClassName)) { - // collision within batch - continue next_probe; - } - } + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - if (result == null) { - break; // we'll try to compile it + if (source == null) { + // If we have no source, then mark every request as a failure. + final UncheckedDeephavenException err = new UncheckedDeephavenException( + "Error Invoking Compiler, no source present in diagnostic:\n" + + diagnostic.getMessage(Locale.getDefault())); + globalFailures.add(err); + return; } - final boolean shouldTrace = shouldTrace(state.fqClassName); - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), - result)) { - if (shouldTrace) { - log.trace().append("Successfully found existing class for ").append(state.fqClassName) - .endl(); - } - state.complete = true; - ++numComplete; - break; - } else if (shouldTrace) { - log.trace().append("Existing class did not match for ").append(state.fqClassName).endl(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); } - } - } + }, + compilerOptions, + classNamesForAnnotationProcessing, + requests.subList(startInclusive, endExclusive).stream() + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); - if (numComplete == requests.size()) { - return; - } + final String compilerOutputText = compilerOutput.toString(); + if (!compilerOutputText.isEmpty()) { + log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); + } - // Couldn't resolve at least one of the requests, so try a round of compilation. - final List compilationRequestAttempts = new ArrayList<>(); - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (!state.complete) { - final QueryCompilerRequest request = requests.get(ii); - compilationRequestAttempts.add(new CompilationRequestAttempt( - request, - state.packageName, - state.fqClassName, - resolvers.get(ii))); - } + if (!globalFailures.isEmpty()) { + final RuntimeException e0 = globalFailures.get(0); + for (int ii = 1; ii < globalFailures.size(); ++ii) { + e0.addSuppressed(globalFailures.get(ii)); } + throw e0; + } - maybeCreateClasses(compilationRequestAttempts); + final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; - // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it - // ... then give the filesystem some time. All requests should use the same deadline. - final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { + // Define compiled classes into a per-batch classloader + final Map compiledClasses = outputFm.getCompiledClasses(); + log.info().append("compilation produced ").append(compiledClasses.size()) + .append(" classes, numFailures=").append(numFailures.get()) + .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") + .endl(); + if (!compiledClasses.isEmpty()) { + // Use the current thread's context classloader as parent so that Groovy-defined classes + // (which may have been loaded after this QueryCompilerImpl was constructed) are visible. + final ClassLoader currentContextCl = Thread.currentThread().getContextClassLoader(); + final ClassLoader batchParent = currentContextCl != null ? currentContextCl : parentClassLoader; + final BatchClassLoader batchCl = new BatchClassLoader(batchParent, compiledClasses); + + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (request.resolver.getFuture().isDone()) { + // already failed continue; } - final QueryCompilerRequest request = requests.get(ii); - final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); - if (resolver.getFuture().isDone()) { - state.complete = true; - ++numComplete; + if (!compiledClasses.containsKey(request.fqClassName)) { + if (wantRetry) { + toRetry.add(request); + } continue; } - // This request may have: - // A. succeeded - // B. Lost a race to another process on the same file system which is compiling the identical formula - // C. Lost a race to another process on the same file system compiling a different formula that collides - - final boolean shouldTrace = shouldTrace(state.fqClassName); - Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - if (clazz == null) { - final long waitStart = System.currentTimeMillis(); - if (shouldTrace) { - log.trace().append("Class not loadable immediately after compile;" - + " entering retry loop for ").append(state.fqClassName).endl(); - } - try { - while (clazz == null && System.currentTimeMillis() < deadline) { - // noinspection BusyWait - Thread.sleep(CODEGEN_LOOP_DELAY_MS); - clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - } - } catch (final InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); - } - if (shouldTrace) { - final long waited = System.currentTimeMillis() - waitStart; - if (clazz != null) { - log.trace().append("Class loaded after waiting ").append(waited) - .append(" ms: ").append(state.fqClassName).endl(); - } else { - log.trace().append("Retry loop timed out after ").append(waited) - .append(" ms: ").append(state.fqClassName).endl(); - } - } + // Load the top-level class (which triggers loading of inner/anonymous classes as needed) + final Class clazz; + try { + clazz = batchCl.loadClass(request.fqClassName); + } catch (ClassNotFoundException e) { + request.resolver.completeExceptionally(new UncheckedDeephavenException( + "Failed to load compiled class: " + request.fqClassName, e)); + continue; } - // However, regardless of A-C, there will be *some* class being found - if (clazz == null) { - throw new IllegalStateException("Unable to load class after delay of " + CODEGEN_TIMEOUT_MS - + ". state index=" + ii + ", fqClassName=" + state.fqClassName + ", parameterClasses" - + request.parameterClasses() + ", destination=" + getClassDestination().getAbsolutePath()); + // Validate the identifying field + final String identifyingFieldValue = loadIdentifyingField(clazz); + if (!request.request.classBody().equals(identifyingFieldValue)) { + request.resolver.completeExceptionally(new IllegalStateException( + "Compiled class body validation failed for " + request.fqClassName)); + continue; } - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { - if (shouldTrace) { - log.trace().append("Loaded matching compiled class ").append(state.fqClassName).endl(); - } - state.complete = true; - ++numComplete; - } else if (shouldTrace) { - log.trace().append("Existing class did not match for ").append(state.fqClassName).endl(); + // Notify caller with codeLog if requested + request.request.codeLog().ifPresent( + sb -> sb.append(makeFinalCode( + request.request.className(), request.request.classBody(), request.packageName))); + + // Complete the future + log.info().append("Resolving ").append(request.fqClassName).endl(); + request.resolver.complete(clazz); + + // Canonicalize the knownClasses entry + synchronized (this) { + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, request.resolver.getFuture()); } } - } - } - - private boolean completeIfResultMatchesQueryCompilerRequest( - final String packageName, - final QueryCompilerRequest request, - final CompletionStageFuture.Resolver> resolver, - final Class result) { - final String identifyingFieldValue = loadIdentifyingField(result); - if (!request.classBody().equals(identifyingFieldValue)) { - final LogEntry logOutput = log.trace().append("Hash collision for ").append(result.getName()) - .append(": loaded identifying field length="); - if (identifyingFieldValue == null) { - logOutput.append("null"); - } else { - logOutput.append(identifyingFieldValue.length()); + } else { + // No output at all - if there are non-failed requests, they need retry + for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { + if (!request.resolver.getFuture().isDone() && wantRetry) { + toRetry.add(request); + } } - logOutput.append(", expected class body length=").append(request.classBody().length()).endl(); - return false; } - // If the caller wants a textual copy of the code we either made, or just found in the cache. - request.codeLog() - .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + return wantRetry && !toRetry.isEmpty(); + } - // If the class we found was indeed the class we were looking for, then complete the future and return it. - resolver.complete(result); + // --- Classpath construction --- - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resolver.getFuture()); + private String getClassPath() { + final StringBuilder sb = new StringBuilder(getJavaClassPath()); + if (additionalClassPathDir != null) { + sb.append(File.pathSeparator).append(additionalClassPathDir.getAbsolutePath()); } - - return true; + return sb.toString(); } - private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { - final boolean shouldTrace = shouldTrace(fqClassName); - try { - if (shouldTrace) { - log.trace().append("Attempting to load ").append(fqClassName) - .append(" with parameter classes ") - .append(LogOutput.STRING_COLLECTION_FORMATTER, parameterClasses.keySet()).endl(); - } - return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); - } catch (ClassNotFoundException cnfe) { - if (shouldTrace) { - log.trace().append("Class not found for ").append(fqClassName).endl(); - } - return null; - } - } + // --- BatchClassLoader --- - private static String loadIdentifyingField(Class c) { - try { - final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); - return (String) field.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Malformed class in cache", e); - } - } + /** + * A classloader that defines classes from a map of name→bytes. All classes from a single compilation batch are + * loaded together. The parent classloader handles all other class resolution. + */ + private static class BatchClassLoader extends ClassLoader { + private final Map classBytes; - private static String makeFinalCode(String className, String classBody, String packageName) { - if (classBody.contains("$CLASSNAME$")) { - throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" - + " the final class name affects the compiled byte code and therefore cannot be dynamically " - + "replaced."); + BatchClassLoader(@NotNull ClassLoader parent, @NotNull Map classBytes) { + super(parent); + this.classBytes = classBytes; } - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; + @Override + protected Class findClass(String name) throws ClassNotFoundException { + final byte[] bytes = classBytes.get(name); + if (bytes != null) { + return defineClass(name, bytes, 0, bytes.length); + } + throw new ClassNotFoundException(name); + } } + // --- InMemoryOutputFileManager --- + /** - * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping - * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java - * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. + * A forwarding JavaFileManager that intercepts class output, storing compiled bytecode in memory rather than + * writing to the filesystem. All other operations are delegated. */ - public static String createEscapedJoinedString(final String originalString) { - return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); - } + private static class InMemoryOutputFileManager extends ForwardingJavaFileManager { + private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); - public static String createEscapedJoinedString(final String originalString, int maxStringLength) { - final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - - // Turn each split into a Java source string by escaping it and surrounding it with " - for (int ii = 0; ii < splits.length; ++ii) { - final String escaped = StringEscapeUtils.escapeJava(splits[ii]); - splits[ii] = "\"" + escaped + "\""; + InMemoryOutputFileManager(JavaFileManager delegate) { + super(delegate); + } + Map getCompiledClasses() { + final Map result = new HashMap<>(); + for (Map.Entry entry : outputClasses.entrySet()) { + result.put(entry.getKey(), entry.getValue().getBytes()); + } + return result; } - assert splits.length > 0; - if (splits.length == 1) { - return splits[0]; + + @Override + public JavaFileObject getJavaFileForOutput( + Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { + final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); + outputClasses.put(className, fileObject); + return fileObject; } - final String formattedInnards = String.join(",\n", splits); - return "String.join(\"\", " + formattedInnards + ")"; - } - private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { - final List splits = new ArrayList<>(); - // exclusive end position of the previous substring. - int previousEnd = 0; - // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. - int currentByteCount = 0; - for (int ii = 0; ii < originalString.length(); ++ii) { - final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); - if (currentByteCount + bytesConsumed > maxBytes) { - // This character won't fit in this string, so we flush the buffer. - splits.add(originalString.substring(previousEnd, ii)); - previousEnd = ii; - currentByteCount = 0; + @Override + public boolean hasLocation(Location location) { + if (location == StandardLocation.CLASS_OUTPUT) { + return true; } - currentByteCount += bytesConsumed; + return super.hasLocation(location); } - // At the end of the loop, either - // 1. there are one or more characters that still need to be added to splits - // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits - splits.add(originalString.substring(previousEnd)); - return splits.toArray(String[]::new); } - private static int calcBytesConsumed(final char ch) { - if (ch == 0) { - return 2; + // --- InMemoryClassFileObject --- + + private static class InMemoryClassFileObject extends SimpleJavaFileObject { + private final String className; + private ByteArrayOutputStream outputStream; + + InMemoryClassFileObject(String className) { + super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); + this.className = className; } - if (ch <= 0x7f) { - return 1; + + @Override + public OutputStream openOutputStream() { + outputStream = new ByteArrayOutputStream(); + return outputStream; } - if (ch <= 0x7ff) { - return 2; + + byte[] getBytes() { + if (outputStream == null) { + throw new IllegalStateException("No bytes available for " + className); + } + return outputStream.toByteArray(); } - return 3; } + // --- Source representation --- + private static class JavaSourceFromString extends SimpleJavaFileObject { final String description; final String code; @@ -959,6 +740,7 @@ private static class JavaSourceFromString extends SimpleJavaFileObject { this.resolver = resolver; } + @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) { return code; } @@ -969,7 +751,6 @@ private static class CompilationRequestAttempt { final String fqClassName; final String finalCode; final String packageName; - final String[] splitPackageName; final QueryCompilerRequest request; final CompletionStageFuture.Resolver> resolver; @@ -988,273 +769,88 @@ private CompilationRequestAttempt( if (logEnabled) { log.info().append("Generating code ").append(finalCode).endl(); - } else if (shouldTrace(fqClassName)) { - log.trace().append("Generating code for ").append(fqClassName).endl(); - } - - splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - final Exception err = new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); - resolver.completeExceptionally(err); } } - public void ensureDirectories(@NotNull final String rootPath) { - if (splitPackageName.length == 0) { - // we've already failed - return; - } - - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); - final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - QueryCompilerImpl.ensureDirectories(rpf, - () -> "Couldn't create package directories: " + rootPathWithPackage); - } - - public JavaSourceFromString makeSource() { + JavaSourceFromString makeSource() { return new JavaSourceFromString(description, fqClassName, finalCode, resolver); } } - private void maybeCreateClasses( - @NotNull final List requests) { - // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package - // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. - // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 - // This temp directory will be where the compiler drops files into, e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class - // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to - // /tmp/workspace/cache/classes/io/deephaven/test - // Note: for this atomic move to work, this temp directory must be on the same file system as the destination - // directory. - final String rootPathAsString; - final String tempDirAsString; - try { - rootPathAsString = getClassDestination().getAbsolutePath(); - for (final CompilationRequestAttempt request : requests) { - request.ensureDirectories(rootPathAsString); - } - - final Path tempPath = - Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); - tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - Exception err = new UncheckedIOException(ioe); - for (final CompilationRequestAttempt request : requests) { - request.resolver.completeExceptionally(err); - } - return; - } - - final ExecutionContext executionContext = ExecutionContext.getContext(); - final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); - - final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - - final int numTasks; - final JobScheduler jobScheduler; - - final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); - if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { - numTasks = 1; - jobScheduler = new ImmediateJobScheduler(); - } else { - numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; - jobScheduler = new OperationInitializerJobScheduler(); - } - - log.trace().append("maybeCreateClasses: ").append(requests.size()).append(" requests, ") - .append(numTasks).append(" tasks, ").append(requestsPerTask) - .append(" requests per task, tempDir=").append(tempDirAsString).endl(); - - final JavaFileManager fileManager = acquireFileManager(); - final AtomicReference exception = new AtomicReference<>(); - final CountDownLatch latch = new CountDownLatch(1); - final Runnable cleanup = () -> { - try { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here - } - try { - releaseFileManager(fileManager); - } catch (Exception e) { - // ignore errors here - } - } finally { - latch.countDown(); - } - }; - - final Consumer onError = err -> { - if (err instanceof RuntimeException) { - exception.set((RuntimeException) err); - } else { - exception.set(new UncheckedDeephavenException("Error during compilation", err)); - } - cleanup.run(); - }; - - jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, - 0, numTasks, (context, jobId, nestedErrorConsumer) -> { - final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - doCreateClasses( - fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); - }, - () -> { - }, - cleanup, - onError); + // --- Utilities --- + private static String loadIdentifyingField(Class c) { try { - latch.await(); - final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); - if (perfEntry != null) { - QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); - } - final RuntimeException err = exception.get(); - if (err != null) { - throw err; - } - } catch (final InterruptedException e) { - throw new CancellationException("interrupted while compiling"); + final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); + return (String) field.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Malformed class in cache", e); } } - private void doCreateClasses( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive) { - final List toRetry = new ArrayList<>(); - // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The - // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to - // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = doCreateClassesSingleRound(fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive, toRetry); - if (!wantRetry) { - return; + static String makeFinalCode(String className, String classBody, String packageName) { + if (classBody.contains("$CLASSNAME$")) { + throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" + + " the final class name affects the compiled byte code and therefore cannot be dynamically " + + "replaced."); } - final List ignored = new ArrayList<>(); - if (doCreateClassesSingleRound(fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), - ignored)) { - // We only retried compilation units that did not fail on the first pass, so we should not have any failures - // on the second pass. - throw new IllegalStateException("Unexpected failure during second pass of compilation"); - } + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; } - private boolean doCreateClassesSingleRound( - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive, - List toRetry) { - final StringWriter compilerOutput = new StringWriter(); - - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList( - "-d", tempDirAsString, - "-cp", classPathAsString, - // this option allows the compiler to attempt to process all source files even if some of them fail - "--should-stop=ifError=GENERATE"); - - final MutableInt numFailures = new MutableInt(0); - final List globalFailures = new ArrayList<>(); - compiler.getTask(compilerOutput, - fileManager, - diagnostic -> { - if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { - return; - } - - final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); - - if (source == null) { - // If we have no source, then mark every request as a failure. - final UncheckedDeephavenException err = new UncheckedDeephavenException( - "Error Invoking Compiler, no source present in diagnostic:\n" - + diagnostic.getMessage(Locale.getDefault())); - globalFailures.add(err); - return; - } + /** + * Transform a string into the corresponding Java source code that compiles into that string. + */ + public static String createEscapedJoinedString(final String originalString) { + return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + } - final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " - + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); - if (source.resolver.completeExceptionally(err)) { - // only count the first failure for each source - numFailures.increment(); - } - }, - compilerOptions, - classNamesForAnnotationProcessing, - requests.subList(startInclusive, endExclusive).stream() - .map(CompilationRequestAttempt::makeSource) - .collect(Collectors.toList())) - .call(); + public static String createEscapedJoinedString(final String originalString, int maxStringLength) { + final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - final String compilerOutputText = compilerOutput.toString(); - if (!compilerOutputText.isEmpty()) { - log.trace().append("Compiler output:\n").append(compilerOutputText).endl(); + for (int ii = 0; ii < splits.length; ++ii) { + final String escaped = StringEscapeUtils.escapeJava(splits[ii]); + splits[ii] = "\"" + escaped + "\""; } - - if (!globalFailures.isEmpty()) { - final RuntimeException e0 = globalFailures.get(0); - for (int ii = 1; ii < globalFailures.size(); ++ii) { - e0.addSuppressed(globalFailures.get(ii)); - } - throw e0; + assert splits.length > 0; + if (splits.length == 1) { + return splits[0]; } + final String formattedInnards = String.join(",\n", splits); + return "String.join(\"\", " + formattedInnards + ")"; + } - final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; - - // The above has compiled into e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various - // class files} - // We want to atomically move it to e.g. - // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - requests.subList(startInclusive, endExclusive).forEach(request -> { - final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); - final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - if (shouldTrace(request.fqClassName)) { - log.trace().append("Successfully moved ").append(srcDir.toString()).append(" to ") - .append(destDir.toString()).endl(); - } - } catch (IOException ioe) { - // The name "isDone" might be misleading here. We haven't called "complete" on the successful - // futures yet, so the only way they would be "done" at this point is if they completed - // exceptionally. - final boolean hasException = request.resolver.getFuture().isDone(); - - if (wantRetry && !Files.exists(srcDir) && !hasException) { - // The move failed, the source directory does not exist, and this compilation unit actually - // succeeded. However, it was not written because some other compilation unit failed. Let's schedule - // this work to try again. - toRetry.add(request); - return; - } - - if (!Files.exists(destDir) && !hasException) { - // Propagate an error here only if the destination does not exist; ignoring issues related to - // collisions with another process. - request.resolver.completeExceptionally(new UncheckedIOException( - "Move failed for some reason other than destination already existing", ioe)); - } + private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { + final List splits = new ArrayList<>(); + int previousEnd = 0; + int currentByteCount = 0; + for (int ii = 0; ii < originalString.length(); ++ii) { + final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); + if (currentByteCount + bytesConsumed > maxBytes) { + splits.add(originalString.substring(previousEnd, ii)); + previousEnd = ii; + currentByteCount = 0; } - }); + currentByteCount += bytesConsumed; + } + splits.add(originalString.substring(previousEnd)); + return splits.toArray(String[]::new); + } - return wantRetry && !toRetry.isEmpty(); + private static int calcBytesConsumed(final char ch) { + if (ch == 0) { + return 2; + } + if (ch <= 0x7f) { + return 1; + } + if (ch <= 0x7ff) { + return 2; + } + return 3; } /** @@ -1331,3 +927,5 @@ private static String getJavaClassPath() { return javaClasspath; } } + + diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 7b50edb5dd7..291b97362ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; @@ -478,7 +478,7 @@ protected void generateFilterCode( .description("Filter Expression: " + formula) .className(CLASS_NAME) .classBody(this.classBody) - .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index d8ccb78e2a0..7a4c4174d32 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -8,7 +8,7 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; @@ -367,7 +367,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - final String joinedFormulaString = InMemoryQueryCompiler.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -817,7 +817,7 @@ private void compileFormula(@NotNull final QueryCompilerRequestProcessor compila .description("Formula Expression: " + formulaString) .className(FORMULA_CLASS_NAME) .classBody(classBody) - .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()).thenApply(clazz -> { try { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java index f7571545c4e..54d4ac2f837 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java @@ -5,7 +5,7 @@ import groovy.lang.Closure; import groovy.lang.GroovyObject; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.util.type.TypeUtils; import java.lang.reflect.Modifier; @@ -51,7 +51,7 @@ private static void visitParameterClass(final Map> found, Class } final String name = cls.getName(); - if (!name.startsWith(InMemoryQueryCompiler.DYNAMIC_CLASS_PREFIX)) { + if (!name.startsWith(QueryCompilerImpl.DYNAMIC_CLASS_PREFIX)) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index 31fc3c6678d..4ea80198ead 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -4,7 +4,7 @@ package io.deephaven.engine.table.impl.select.codegen; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.CompletionStageFuture; @@ -50,7 +50,7 @@ public static CompletionStageFuture create( .description("FormulaKernel: " + originalFormulaString) .className(CLASS_NAME) .classBody(classBody) - .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()).thenApply(clazz -> { final FormulaKernelFactory fkf; try { @@ -242,7 +242,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = InMemoryQueryCompiler.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index 8e7fcab31eb..dfe085504f3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -95,7 +95,7 @@ protected AbstractScriptSession( this.classCacheDirectory = classCacheDirectory; queryScope = new ScriptSessionQueryScope(); - final QueryCompiler compilerContext = InMemoryQueryCompiler.create(classCacheDirectory); + final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory); executionContext = ExecutionContext.newBuilder() .markSystemic() diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index c7a6469300e..6bdeeecd5c9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -4,7 +4,7 @@ package io.deephaven.engine.util; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import java.util.*; @@ -60,7 +60,7 @@ public static Supplier compileSimpleFunction(final Class res .description("Simple Function: " + code) .className(className) .classBody(classBody.toString()) - .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()); try { @@ -86,7 +86,7 @@ public static Class getClassThroughCompilation(final String object) { .description("Formula: return " + object + ".class") .className(className) .classBody(classBody.toString()) - .packageNameRoot(InMemoryQueryCompiler.FORMULA_CLASS_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()); try { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 3a4f8926154..477290ceff3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -81,7 +81,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession strValues = stringCol("StrValue", "hello", "world"); final Table source = TableTools.newTable(strValues); - InMemoryQueryCompiler.setLogEnabled(true); + QueryCompilerImpl.setLogEnabled(true); final Table updated = source.update("P=pred.test(StrValue)"); assertTableEquals(TableTools.newTable(strValues, booleanCol("P", true, false)), updated); final Table filtered = source.where("pred.test(StrValue)"); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index 9269fa69633..ca28d3cebe8 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(InMemoryQueryCompiler.create(null)) + .setQueryCompiler(QueryCompilerImpl.create(null)) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java index d16178a6bf0..fa68c7d56cb 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.util.pools.ChunkPoolReleaseTracking; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.TestExecutionContext; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -86,7 +86,7 @@ public void setUp() throws Exception { delayedErrors.clear(); livenessScopeCloseable = LivenessScopeStack.open(new LivenessScope(true), true); - oldLogEnabled = InMemoryQueryCompiler.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); + oldLogEnabled = QueryCompilerImpl.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); oldSerialSafe = updateGraph.setSerialTableOperationsSafe(true); AsyncErrorLogger.init(); ChunkPoolReleaseTracking.enableStrict(); @@ -127,7 +127,7 @@ public void tearDown() throws Exception { } final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.setSerialTableOperationsSafe(oldSerialSafe); - InMemoryQueryCompiler.setLogEnabled(oldLogEnabled); + QueryCompilerImpl.setLogEnabled(oldLogEnabled); // reset the execution context executionContext.close(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index a1d24d57935..8409453f0bd 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -5,7 +5,7 @@ import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.Table; @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - InMemoryQueryCompiler.create(null)) + QueryCompilerImpl.create(null)) .build(); exContextCloseable = context.open(); diff --git a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java index f50446f12af..c7d598e4221 100644 --- a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java +++ b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java @@ -9,7 +9,7 @@ import io.deephaven.auth.AuthContext; import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.InMemoryQueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.liveness.StandaloneLivenessManager; import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfLong; @@ -56,7 +56,7 @@ public void before() throws IOException { cacheDir.deleteOnExit(); executionContext = ExecutionContext.newBuilder().newQueryLibrary().newQueryScope() - .setQueryCompiler(InMemoryQueryCompiler.create(null)) + .setQueryCompiler(QueryCompilerImpl.create(null)) .setOperationInitializer(ForkJoinPoolOperationInitializer.fromCommonPool()) .setUpdateGraph(defaultUpdateGraph).build().withAuthContext(new AuthContext.Anonymous()); } From 2d96a76958a4af48e9327edb4dd064abe37530bb Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 25 Jun 2026 10:57:12 -0500 Subject: [PATCH 05/12] Delete failed attempt --- .../context/InMemoryQueryCompiler.java.bak | 722 ------------------ 1 file changed, 722 deletions(-) delete mode 100644 engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak diff --git a/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak b/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak deleted file mode 100644 index 4fc45cac53a..00000000000 --- a/engine/context/src/main/java/io/deephaven/engine/context/InMemoryQueryCompiler.java.bak +++ /dev/null @@ -1,722 +0,0 @@ -// -// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending -// -// package io.deephaven.engine.context; -// -// import io.deephaven.UncheckedDeephavenException; -// import io.deephaven.configuration.Configuration; -// import io.deephaven.engine.context.util.SynchronizedJavaFileManager; -// import io.deephaven.internal.log.LoggerFactory; -// import io.deephaven.io.logger.Logger; -// import io.deephaven.util.ByteUtils; -// import io.deephaven.util.CompletionStageFuture; -// import io.deephaven.util.mutable.MutableInt; -// import org.apache.commons.text.StringEscapeUtils; -// import org.jetbrains.annotations.NotNull; -// import org.jetbrains.annotations.Nullable; -// -// import javax.tools.*; -// import java.io.*; -// import java.lang.reflect.Field; -// import java.net.MalformedURLException; -// import java.net.URI; -// import java.net.URL; -// import java.net.URLClassLoader; -// import java.nio.charset.StandardCharsets; -// import java.security.MessageDigest; -// import java.security.NoSuchAlgorithmException; -// import java.util.*; -// import java.util.concurrent.*; -// import java.util.concurrent.atomic.AtomicReference; -// import java.util.stream.Collectors; -// import java.util.stream.Stream; -// -///** -// * A {@link QueryCompiler} implementation that compiles Java source to bytecode in memory (no filesystem output). -// *

-// * Like the filesystem-backed {@link QueryCompilerImpl}, this implementation uses {@code java.class.path} plus an -// * optional extra class directory for resolving compilation dependencies (e.g., Groovy-generated bytecode). The key -// * difference is that compiled classes are defined directly into a classloader rather than written to disk. -// *

-// *

Key invariants

-// *
    -// *
  • No formula compiled by this compiler ever depends on another formula compiled by this or any other -// * QueryCompiler.
    • -// *
  • The classloader is created as a child of the provided parent classloader.
    • -// *
  • Groovy/dynamic classes must be written to the extra class directory before formulas referencing them are -// * compiled.
    • -// *
-// */ -// public class InMemoryQueryCompiler implements QueryCompiler { -// -// private static final Logger log = LoggerFactory.getLogger(InMemoryQueryCompiler.class); -// -// private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; -// private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); -// private static final int MAX_CLASS_COLLISIONS = 128; -// private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; -// -// public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; -// public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; -// -// private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); -// -// private static JavaCompiler compiler; -// private static final AtomicReference standardFileManagerCache = new AtomicReference<>(); -// -// private static void ensureJavaCompiler() { -// synchronized (InMemoryQueryCompiler.class) { -// if (compiler == null) { -// compiler = ToolProvider.getSystemJavaCompiler(); -// if (compiler == null) { -// throw new UncheckedDeephavenException( -// "No Java compiler provided - are you using a JRE instead of a JDK?"); -// } -// } -// } -// } -// -// private static JavaFileManager acquireFileManager() { -// JavaFileManager fm = standardFileManagerCache.getAndSet(null); -// if (fm == null) { -// fm = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); -// } -// return fm; -// } -// -// private static void releaseFileManager(@NotNull final JavaFileManager fm) { -// if (!standardFileManagerCache.compareAndSet(null, fm)) { -// try { -// fm.close(); -// } catch (final IOException err) { -// throw new UncheckedIOException("Could not close JavaFileManager", err); -// } -// } -// } -// -// // ---- Instance state ---- -// -// private final Map>> knownClasses = new HashMap<>(); -// private final File extraClassDirectory; -// private final InMemoryClassLoader classLoader; -// private final List classNamesForAnnotationProcessing; -// -// /** -// * Creates a new InMemoryQueryCompiler. -// * -// * @param extraClassDirectory optional directory containing additional class files (e.g., Groovy bytecode) -// * @param parentClassLoader the parent class loader -// */ -// public static InMemoryQueryCompiler create( -// @Nullable final File extraClassDirectory, -// @NotNull final ClassLoader parentClassLoader) { -// return new InMemoryQueryCompiler(extraClassDirectory, parentClassLoader, null); -// } -// -// /** -// * Creates a new InMemoryQueryCompiler using the current thread's context classloader. -// * -// * @param extraClassDirectory optional extra class directory for compilation dependencies -// */ -// public static InMemoryQueryCompiler create(@Nullable final File extraClassDirectory) { -// return new InMemoryQueryCompiler(extraClassDirectory, Thread.currentThread().getContextClassLoader(), null); -// } -// -// static InMemoryQueryCompiler createForUnitTests() { -// return createForUnitTests(null); -// } -// -// static InMemoryQueryCompiler createForUnitTests(final List classNamesForAnnotationProcessing) { -// return new InMemoryQueryCompiler(null, InMemoryQueryCompiler.class.getClassLoader(), -// classNamesForAnnotationProcessing); -// } -// -// private InMemoryQueryCompiler( -// @Nullable final File extraClassDirectory, -// @NotNull final ClassLoader parentClassLoader, -// final List classNamesForAnnotationProcessing) { -// ensureJavaCompiler(); -// this.extraClassDirectory = extraClassDirectory; -// -// if (extraClassDirectory != null) { -// try { -// URL[] urls = new URL[] {extraClassDirectory.toURI().toURL()}; -// this.classLoader = new InMemoryClassLoader(urls, parentClassLoader); -// } catch (MalformedURLException e) { -// throw new UncheckedDeephavenException(e); -// } -// } else { -// this.classLoader = new InMemoryClassLoader(new URL[0], parentClassLoader); -// } -// -// this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; -// } -// -// public static boolean setLogEnabled(boolean logEnabled) { -// boolean original = InMemoryQueryCompiler.logEnabled; -// InMemoryQueryCompiler.logEnabled = logEnabled; -// return original; -// } -// -// @Override -// public void compile( -// @NotNull final QueryCompilerRequest[] requests, -// @NotNull final CompletionStageFuture.Resolver>[] resolvers) { -// if (requests.length == 0) { -// return; -// } -// if (requests.length != resolvers.length) { -// throw new IllegalArgumentException("Requests and resolvers must be the same length"); -// } -// -// // noinspection unchecked -// final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; -// -// final List newRequests = new ArrayList<>(); -// final List>> newResolvers = new ArrayList<>(); -// -// synchronized (this) { -// for (int ii = 0; ii < requests.length; ++ii) { -// final QueryCompilerRequest request = requests[ii]; -// final CompletionStageFuture.Resolver> resolver = resolvers[ii]; -// -// CompletionStageFuture> future = -// knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); -// if (future == null) { -// newRequests.add(request); -// newResolvers.add(resolver); -// future = resolver.getFuture(); -// } -// allFutures[ii] = future; -// } -// } -// -// if (!newResolvers.isEmpty()) { -// try { -// compileHelper(newRequests, newResolvers); -// } catch (RuntimeException e) { -// synchronized (this) { -// for (int ii = 0; ii < newRequests.size(); ++ii) { -// if (newResolvers.get(ii).completeExceptionally(e)) { -// knownClasses.remove(newRequests.get(ii).classBody()); -// } -// } -// } -// throw e; -// } -// } -// -// for (int ii = 0; ii < requests.length; ++ii) { -// try { -// resolvers[ii].complete(allFutures[ii].get()); -// } catch (ExecutionException err) { -// resolvers[ii].completeExceptionally(err.getCause()); -// } catch (InterruptedException err) { -// resolvers[ii].completeExceptionally(err); -// } catch (Throwable err) { -// resolvers[ii].completeExceptionally(err); -// } -// } -// } -// -// // ---- Compilation logic ---- -// -// private static class CompilationState { -// int nextProbeIndex; -// boolean complete; -// String packageName; -// String fqClassName; -// } -// -// private void compileHelper( -// @NotNull final List requests, -// @NotNull final List>> resolvers) { -// final MessageDigest digest; -// try { -// digest = MessageDigest.getInstance("SHA-256"); -// } catch (NoSuchAlgorithmException e) { -// throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); -// } -// -// final String[] basicHashText = new String[requests.size()]; -// for (int ii = 0; ii < requests.size(); ++ii) { -// basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( -// requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); -// } -// -// int numComplete = 0; -// final CompilationState[] states = new CompilationState[requests.size()]; -// for (int ii = 0; ii < requests.size(); ++ii) { -// states[ii] = new CompilationState(); -// } -// -// while (numComplete < requests.size()) { -// for (int ii = 0; ii < requests.size(); ++ii) { -// final CompilationState state = states[ii]; -// if (state.complete) { -// continue; -// } -// -// next_probe: while (true) { -// final int pi = state.nextProbeIndex++; -// final String packageNameSuffix = "c_" + basicHashText[ii] -// + (pi == 0 ? "" : ("p" + pi)) -// + "v" + JAVA_CLASS_VERSION; -// -// final QueryCompilerRequest request = requests.get(ii); -// if (pi >= MAX_CLASS_COLLISIONS) { -// Exception err = new IllegalStateException("Found too many collisions for package name root " -// + request.packageNameRoot() + ", class name=" + request.className() + ", class body " -// + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); -// resolvers.get(ii).completeExceptionally(err); -// state.complete = true; -// ++numComplete; -// break; -// } -// -// state.packageName = request.getPackageName(packageNameSuffix); -// state.fqClassName = state.packageName + "." + request.className(); -// -// for (int jj = 0; jj < ii; ++jj) { -// if (states[jj].fqClassName.equals(state.fqClassName)) { -// continue next_probe; -// } -// } -// -// Class result = tryLoadClassByFqName(state.fqClassName); -// if (result == null) { -// break; -// } -// -// if (completeIfResultMatches(state.packageName, request, resolvers.get(ii), result)) { -// state.complete = true; -// ++numComplete; -// break; -// } -// } -// } -// -// if (numComplete == requests.size()) { -// return; -// } -// -// final List attempts = new ArrayList<>(); -// for (int ii = 0; ii < requests.size(); ++ii) { -// final CompilationState state = states[ii]; -// if (!state.complete) { -// attempts.add(new CompilationRequestAttempt( -// requests.get(ii), state.packageName, state.fqClassName, resolvers.get(ii))); -// } -// } -// -// doCompileAndDefine(attempts); -// -// for (int ii = 0; ii < requests.size(); ++ii) { -// final CompilationState state = states[ii]; -// if (state.complete) { -// continue; -// } -// -// final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); -// if (resolver.getFuture().isDone()) { -// state.complete = true; -// ++numComplete; -// continue; -// } -// -// final QueryCompilerRequest request = requests.get(ii); -// Class clazz = tryLoadClassByFqName(state.fqClassName); -// if (clazz == null) { -// throw new IllegalStateException("Class not found after compilation: " + state.fqClassName); -// } -// -// if (completeIfResultMatches(state.packageName, request, resolver, clazz)) { -// state.complete = true; -// ++numComplete; -// } -// } -// } -// } -// -// private boolean completeIfResultMatches( -// final String packageName, -// final QueryCompilerRequest request, -// final CompletionStageFuture.Resolver> resolver, -// final Class result) { -// final String identifyingFieldValue = loadIdentifyingField(result); -// if (!request.classBody().equals(identifyingFieldValue)) { -// return false; -// } -// -// request.codeLog() -// .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); -// -// resolver.complete(result); -// -// synchronized (this) { -// knownClasses.remove(identifyingFieldValue); -// knownClasses.put(identifyingFieldValue, resolver.getFuture()); -// } -// -// return true; -// } -// -// private Class tryLoadClassByFqName(String fqClassName) { -// try { -// return classLoader.loadClass(fqClassName); -// } catch (ClassNotFoundException cnfe) { -// return null; -// } -// } -// -// private static String loadIdentifyingField(Class c) { -// try { -// final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); -// return (String) field.get(null); -// } catch (NoSuchFieldException | IllegalAccessException e) { -// throw new IllegalStateException("Malformed class in cache", e); -// } -// } -// -// // ---- Compilation and class definition ---- -// -// private void doCompileAndDefine(@NotNull final List requests) { -// final JavaFileManager fileManager = acquireFileManager(); -// try { -// final List toRetry = new ArrayList<>(); -// final boolean wantRetry = compileSingleRound(fileManager, requests, 0, requests.size(), toRetry); -// if (wantRetry) { -// final List ignored = new ArrayList<>(); -// if (compileSingleRound(fileManager, toRetry, 0, toRetry.size(), ignored)) { -// throw new IllegalStateException("Unexpected failure during second pass of compilation"); -// } -// } -// } finally { -// releaseFileManager(fileManager); -// } -// } -// -// private boolean compileSingleRound( -// @NotNull final JavaFileManager fileManager, -// @NotNull final List requests, -// final int startInclusive, -// final int endExclusive, -// @NotNull final List toRetry) { -// -// final InMemoryFileManager inMemoryFm = new InMemoryFileManager(fileManager); -// final StringWriter compilerOutput = new StringWriter(); -// -// final String classPathAsString = getClassPath(); -// final List compilerOptions = Arrays.asList( -// "-cp", classPathAsString, -// "--should-stop=ifError=GENERATE"); -// -// final MutableInt numFailures = new MutableInt(0); -// final List globalFailures = new ArrayList<>(); -// -// compiler.getTask(compilerOutput, -// inMemoryFm, -// diagnostic -> { -// if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { -// return; -// } -// -// final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); -// -// if (source == null) { -// final UncheckedDeephavenException err = new UncheckedDeephavenException( -// "Error Invoking Compiler, no source present in diagnostic:\n" -// + diagnostic.getMessage(Locale.getDefault())); -// globalFailures.add(err); -// return; -// } -// -// final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " -// + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); -// if (source.resolver.completeExceptionally(err)) { -// numFailures.increment(); -// } -// }, -// compilerOptions, -// classNamesForAnnotationProcessing, -// requests.subList(startInclusive, endExclusive).stream() -// .map(CompilationRequestAttempt::makeSource) -// .collect(Collectors.toList())) -// .call(); -// -// if (!globalFailures.isEmpty()) { -// final RuntimeException e0 = globalFailures.get(0); -// for (int ii = 1; ii < globalFailures.size(); ++ii) { -// e0.addSuppressed(globalFailures.get(ii)); -// } -// throw e0; -// } -// -// final boolean wantRetry = numFailures.get() > 0 && numFailures.get() != endExclusive - startInclusive; -// -// final Map compiledClasses = inMemoryFm.getCompiledClasses(); -// for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { -// if (request.resolver.getFuture().isDone()) { -// continue; -// } -// -// if (!compiledClasses.containsKey(request.fqClassName)) { -// if (wantRetry) { -// toRetry.add(request); -// } -// continue; -// } -// -// final String classPrefix = request.fqClassName; -// for (Map.Entry entry : compiledClasses.entrySet()) { -// final String className = entry.getKey(); -// if (className.equals(classPrefix) || className.startsWith(classPrefix + "$")) { -// classLoader.addClass(className, entry.getValue()); -// } -// } -// } -// -// return wantRetry && !toRetry.isEmpty(); -// } -// -// // ---- Classpath ---- -// -// private String getClassPath() { -// final StringBuilder sb = new StringBuilder(System.getProperty("java.class.path")); -// if (extraClassDirectory != null) { -// sb.append(File.pathSeparator).append(extraClassDirectory.getAbsolutePath()); -// } -// -// String result = sb.toString(); -// final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; -// if (result.matches(intellijClassPathJarRegex)) { -// try { -// final Enumeration resources = -// InMemoryQueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); -// final java.util.jar.Attributes.Name createdByAttribute = -// new java.util.jar.Attributes.Name("Created-By"); -// final java.util.jar.Attributes.Name classPathAttribute = -// new java.util.jar.Attributes.Name("Class-Path"); -// while (resources.hasMoreElements()) { -// final java.util.jar.Manifest manifest = -// new java.util.jar.Manifest(resources.nextElement().openStream()); -// final java.util.jar.Attributes attributes = manifest.getMainAttributes(); -// final Object createdBy = attributes.get(createdByAttribute); -// if ("IntelliJ IDEA".equals(createdBy)) { -// final String extendedClassPath = (String) attributes.get(classPathAttribute); -// if (extendedClassPath != null) { -// final String filePaths = Stream.of(extendedClassPath.split("file:/")) -// .map(String::trim) -// .filter(fileName -> !fileName.isEmpty()) -// .collect(Collectors.joining(File.pathSeparator)); -// result = Stream.of(result.split(File.pathSeparator)) -// .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) -// .collect(Collectors.joining(File.pathSeparator)); -// } -// } -// } -// } catch (IOException e) { -// throw new UncheckedIOException("Error extracting manifest file from classpath.\n", e); -// } -// } -// return result; -// } -// -// // ---- InMemoryClassLoader ---- -// -// /** -// * A URLClassLoader that also supports defining classes from in-memory bytecode. -// * The URL[] includes the extra class directory so parent delegation finds Groovy classes. -// */ -// private static class InMemoryClassLoader extends URLClassLoader { -// -// private final ConcurrentHashMap inMemoryClasses = new ConcurrentHashMap<>(); -// -// InMemoryClassLoader(URL[] urls, ClassLoader parent) { -// super(urls, parent); -// } -// -// void addClass(String name, byte[] bytes) { -// inMemoryClasses.put(name, bytes); -// } -// -// @Override -// protected Class findClass(String name) throws ClassNotFoundException { -// final byte[] bytes = inMemoryClasses.remove(name); -// if (bytes != null) { -// return defineClass(name, bytes, 0, bytes.length); -// } -// return super.findClass(name); -// } -// } -// -// // ---- InMemoryFileManager ---- -// -// /** -// * A forwarding JavaFileManager that intercepts class output, writing to in-memory byte arrays. -// */ -// private static class InMemoryFileManager extends ForwardingJavaFileManager { -// -// private final ConcurrentHashMap outputClasses = new ConcurrentHashMap<>(); -// -// InMemoryFileManager(JavaFileManager delegate) { -// super(delegate); -// } -// -// Map getCompiledClasses() { -// final Map result = new HashMap<>(); -// for (Map.Entry entry : outputClasses.entrySet()) { -// result.put(entry.getKey(), entry.getValue().getBytes()); -// } -// return result; -// } -// -// @Override -// public JavaFileObject getJavaFileForOutput( -// Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { -// final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); -// outputClasses.put(className, fileObject); -// return fileObject; -// } -// } -// -// // ---- InMemoryClassFileObject ---- -// -// private static class InMemoryClassFileObject extends SimpleJavaFileObject { -// private final String className; -// private ByteArrayOutputStream outputStream; -// -// InMemoryClassFileObject(String className) { -// super(URI.create("mem:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); -// this.className = className; -// } -// -// @Override -// public OutputStream openOutputStream() { -// outputStream = new ByteArrayOutputStream(); -// return outputStream; -// } -// -// byte[] getBytes() { -// if (outputStream == null) { -// throw new IllegalStateException("No bytes available for " + className); -// } -// return outputStream.toByteArray(); -// } -// } -// -// // ---- Source representation ---- -// -// private static class JavaSourceFromString extends SimpleJavaFileObject { -// final String description; -// final String code; -// final CompletionStageFuture.Resolver> resolver; -// -// JavaSourceFromString(String description, String name, String code, -// CompletionStageFuture.Resolver> resolver) { -// super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); -// this.description = description; -// this.code = code; -// this.resolver = resolver; -// } -// -// @Override -// public CharSequence getCharContent(boolean ignoreEncodingErrors) { -// return code; -// } -// } -// -// private static class CompilationRequestAttempt { -// final String description; -// final String fqClassName; -// final String finalCode; -// final String packageName; -// final QueryCompilerRequest request; -// final CompletionStageFuture.Resolver> resolver; -// -// CompilationRequestAttempt( -// @NotNull final QueryCompilerRequest request, -// @NotNull final String packageName, -// @NotNull final String fqClassName, -// @NotNull final CompletionStageFuture.Resolver> resolver) { -// this.description = request.description(); -// this.fqClassName = fqClassName; -// this.resolver = resolver; -// this.packageName = packageName; -// this.request = request; -// this.finalCode = makeFinalCode(request.className(), request.classBody(), packageName); -// -// if (logEnabled) { -// log.info().append("Generating code ").append(finalCode).endl(); -// } -// } -// -// JavaSourceFromString makeSource() { -// return new JavaSourceFromString(description, fqClassName, finalCode, resolver); -// } -// } -// -// // ---- Code generation utilities ---- -// -// static String makeFinalCode(String className, String classBody, String packageName) { -// if (classBody.contains("$CLASSNAME$")) { -// throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" -// + " the final class name affects the compiled byte code and therefore cannot be dynamically " -// + "replaced."); -// } -// -// final String joinedEscapedBody = createEscapedJoinedString(classBody); -// classBody = classBody.substring(0, classBody.lastIndexOf("}")); -// classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; -// return "package " + packageName + ";\n" + classBody; -// } -// -// public static String createEscapedJoinedString(final String originalString) { -// return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); -// } -// -// public static String createEscapedJoinedString(final String originalString, int maxStringLength) { -// final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); -// for (int ii = 0; ii < splits.length; ++ii) { -// final String escaped = StringEscapeUtils.escapeJava(splits[ii]); -// splits[ii] = "\"" + escaped + "\""; -// } -// if (splits.length == 1) { -// return splits[0]; -// } -// final String formattedInnards = String.join(",\n", splits); -// return "String.join(\"\", " + formattedInnards + ")"; -// } -// -// private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { -// final List splits = new ArrayList<>(); -// int previousEnd = 0; -// int currentByteCount = 0; -// for (int ii = 0; ii < originalString.length(); ++ii) { -// final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); -// if (currentByteCount + bytesConsumed > maxBytes) { -// splits.add(originalString.substring(previousEnd, ii)); -// previousEnd = ii; -// currentByteCount = 0; -// } -// currentByteCount += bytesConsumed; -// } -// splits.add(originalString.substring(previousEnd)); -// return splits.toArray(String[]::new); -// } -// -// private static int calcBytesConsumed(final char ch) { -// if (ch == 0) { -// return 2; -// } -// if (ch <= 0x7f) { -// return 1; -// } -// if (ch <= 0x7ff) { -// return 2; -// } -// return 3; -// } -// -// } -// From 8272e74e8a00af4ce1a8ababe149dc01c3e04601 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 25 Jun 2026 16:13:15 -0500 Subject: [PATCH 06/12] more cleanup, some ai assisted --- .../engine/context/QueryCompilerImpl.java | 109 ++++++------------ .../engine/context/TestExecutionContext.java | 2 +- .../parquet/table/TableWriteBenchmark.java | 2 +- .../impl/util/TestUpdateAncestorViz.java | 2 +- 4 files changed, 40 insertions(+), 75 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 24e330977a5..b1e28839897 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -50,16 +50,7 @@ * The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Class nor this - * compiler instance are reachable. - * - *

Constraints

- *
    - *
  • Compiled classes must not depend on other classes compiled by this or any other QueryCompiler instance.
    • - *
  • The caller must ensure the parent classloader already contains any classes referenced by compiled formulas (e.g., - * Groovy-defined classes).
    • - *
  • Each compilation request produces a single top-level class (matching {@link QueryCompilerRequest#className()}) - * which may contain static inner classes and anonymous classes.
    • - *
+ * compiler instance are reachable. Returned classes then */ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { @@ -154,11 +145,14 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage * * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) */ - public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir) { + public static QueryCompiler create(@Nullable final File additionalClassPathDir) { return new QueryCompilerImpl(additionalClassPathDir, null); } - public static QueryCompilerImpl createForUnitTests() { + /** + * Creates a new compiler that has no extra directory to read from, suitable for unit tests. + */ + public static QueryCompiler createForUnitTests() { return createForUnitTests(null); } @@ -166,12 +160,10 @@ static QueryCompilerImpl createForUnitTests(final List classNamesForAnno return new QueryCompilerImpl(null, classNamesForAnnotationProcessing); } + /** Map from SHA-256 hash of class body → future for the compiled class. Keyed by hash to avoid retaining source. */ private final Map>> knownClasses = new HashMap<>(); - /** Set of fully-qualified class names already assigned, for collision avoidance. */ - private final Set takenNames = new HashSet<>(); - - /** The context classloader captured at construction time; used as parent for per-batch classloaders. */ + /** The context classloader captured at construction time; used as fallback parent for per-batch classloaders. */ private final ClassLoader parentClassLoader; /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ @@ -246,11 +238,25 @@ public void compile( }, requests).endl(); } + // Compute content hashes for deduplication + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); + } + final String[] hashes = new String[requests.length]; + for (int ii = 0; ii < requests.length; ++ii) { + hashes[ii] = ByteUtils.byteArrToHex(digest.digest( + requests[ii].classBody().getBytes(StandardCharsets.UTF_8))); + } + // noinspection unchecked final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; final List newRequests = new ArrayList<>(); final List>> newResolvers = new ArrayList<>(); + final List newHashes = new ArrayList<>(); synchronized (this) { for (int ii = 0; ii < requests.length; ++ii) { @@ -258,10 +264,11 @@ public void compile( final CompletionStageFuture.Resolver> resolver = resolvers[ii]; CompletionStageFuture> future = - knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + knownClasses.putIfAbsent(hashes[ii], resolver.getFuture()); if (future == null) { newRequests.add(request); newResolvers.add(resolver); + newHashes.add(hashes[ii]); future = resolver.getFuture(); } else if (shouldTrace) { log.trace().append("Found existing future in knownClasses for ").append(request.className()) @@ -274,14 +281,14 @@ public void compile( if (!newResolvers.isEmpty()) { // It's my job to fulfill these futures. try { - compileHelper(newRequests, newResolvers); + compileHelper(newRequests, newResolvers, newHashes); } catch (RuntimeException e) { // These failures are not applicable to a single request, so we can't just complete the future and // leave the failure in the cache. synchronized (this) { for (int ii = 0; ii < newRequests.size(); ++ii) { if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newRequests.get(ii).classBody()); + knownClasses.remove(newHashes.get(ii)); } } } @@ -321,62 +328,26 @@ public void compile( private void compileHelper( @NotNull final List requests, - @NotNull final List>> resolvers) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } - - // Assign unique FQ class names for each request + @NotNull final List>> resolvers, + @NotNull final List hashes) { + // Assign unique FQ class names for each request using pre-computed hashes final String[] fqClassNames = new String[requests.size()]; final String[] packageNames = new String[requests.size()]; - synchronized (this) { - for (int ii = 0; ii < requests.size(); ++ii) { - final QueryCompilerRequest request = requests.get(ii); - final String hashText = ByteUtils.byteArrToHex(digest.digest( - request.classBody().getBytes(StandardCharsets.UTF_8))); - - String fqClassName = null; - String packageName = null; - for (int pi = 0; pi < 128; ++pi) { - final String packageNameSuffix = "c_" + hashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - packageName = request.getPackageName(packageNameSuffix); - fqClassName = packageName + "." + request.className(); - if (!takenNames.contains(fqClassName)) { - break; - } - fqClassName = null; - } - if (fqClassName == null) { - resolvers.get(ii).completeExceptionally(new IllegalStateException( - "Unable to assign unique class name for " + request.className())); - continue; - } - takenNames.add(fqClassName); - fqClassNames[ii] = fqClassName; - packageNames[ii] = packageName; - } + for (int ii = 0; ii < requests.size(); ++ii) { + final QueryCompilerRequest request = requests.get(ii); + final String packageNameSuffix = "c_" + hashes.get(ii) + "v" + JAVA_CLASS_VERSION; + packageNames[ii] = request.getPackageName(packageNameSuffix); + fqClassNames[ii] = packageNames[ii] + "." + request.className(); } - // Build compilation attempts for requests that got a valid name + // Build compilation attempts final List attempts = new ArrayList<>(); for (int ii = 0; ii < requests.size(); ++ii) { - if (fqClassNames[ii] == null) { - continue; // already failed - } attempts.add(new CompilationRequestAttempt( requests.get(ii), packageNames[ii], fqClassNames[ii], resolvers.get(ii))); } - if (attempts.isEmpty()) { - return; - } - // Compile and define compileAndDefine(attempts); @@ -556,7 +527,7 @@ private boolean doCompileAndDefineSingleRound( // Define compiled classes into a per-batch classloader final Map compiledClasses = outputFm.getCompiledClasses(); - log.info().append("compilation produced ").append(compiledClasses.size()) + log.trace().append("compilation produced ").append(compiledClasses.size()) .append(" classes, numFailures=").append(numFailures.get()) .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") .endl(); @@ -604,14 +575,8 @@ private boolean doCompileAndDefineSingleRound( request.request.className(), request.request.classBody(), request.packageName))); // Complete the future - log.info().append("Resolving ").append(request.fqClassName).endl(); + log.trace().append("Resolving ").append(request.fqClassName).endl(); request.resolver.complete(clazz); - - // Canonicalize the knownClasses entry - synchronized (this) { - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, request.resolver.getFuture()); - } } } else { // No output at all - if there are non-failed requests, they need retry diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index ca28d3cebe8..da1d636c03b 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompilerImpl.create(null)) + .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index 8409453f0bd..b38c4a78ed5 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompilerImpl.create(null)) + QueryCompilerImpl.createForUnitTests()) .build(); exContextCloseable = context.open(); diff --git a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java index c7d598e4221..d8085d8c409 100644 --- a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java +++ b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java @@ -56,7 +56,7 @@ public void before() throws IOException { cacheDir.deleteOnExit(); executionContext = ExecutionContext.newBuilder().newQueryLibrary().newQueryScope() - .setQueryCompiler(QueryCompilerImpl.create(null)) + .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) .setOperationInitializer(ForkJoinPoolOperationInitializer.fromCommonPool()) .setUpdateGraph(defaultUpdateGraph).build().withAuthContext(new AuthContext.Anonymous()); } From 778cf93bee444b0896bf6f75aee8a764e0877626 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 30 Jun 2026 15:03:01 -0500 Subject: [PATCH 07/12] Mostly manual refactor to simplify new impl --- .../engine/context/QueryCompilerImpl.java | 229 +++++++++++++----- 1 file changed, 175 insertions(+), 54 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index b1e28839897..e319f9a3fcb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -3,6 +3,7 @@ // package io.deephaven.engine.context; +import com.google.common.hash.Hashing; import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; import io.deephaven.base.log.LogOutput; @@ -18,23 +19,51 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; -import io.deephaven.util.ByteUtils; import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import javax.tools.*; -import java.io.*; +import javax.tools.Diagnostic; +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardLocation; +import javax.tools.ToolProvider; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.jar.Attributes; @@ -160,8 +189,94 @@ static QueryCompilerImpl createForUnitTests(final List classNamesForAnno return new QueryCompilerImpl(null, classNamesForAnnotationProcessing); } - /** Map from SHA-256 hash of class body → future for the compiled class. Keyed by hash to avoid retaining source. */ - private final Map>> knownClasses = new HashMap<>(); + /** + * Cache from SHA-256 hash of class body to entry. While compilation is in-flight, the entry holds a future for + * coordination. Once complete, the entry transitions to a weak reference so the class (and its classloader and + * bytecode) can be GC'd when no longer in use. All transitions are atomic via per-entry synchronization. + * Stale entries are cleaned up via a {@link ReferenceQueue} drained at the start of each {@link #compile} call. + */ + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + /** Queue that receives cleared weak references, allowing us to remove stale cache entries. */ + private final ReferenceQueue> staleQueue = new ReferenceQueue<>(); + + /** Drains the stale queue and removes any cache entries whose weak references have been cleared by GC. */ + private void evictStaleEntries() { + KeyedWeakReference ref; + while ((ref = (KeyedWeakReference) staleQueue.poll()) != null) { + cache.remove(ref.key); + } + } + + /** + * A WeakReference that remembers its cache key, so we can remove the entry when the referent is GC'd. + */ + private static final class KeyedWeakReference extends WeakReference> { + final String key; + + KeyedWeakReference(@NotNull Class referent, @NotNull String key, + @NotNull ReferenceQueue> queue) { + super(referent, queue); + this.key = key; + } + } + + /** + * A cache entry that transitions from in-flight (future-based coordination) to weakly-cached (reclaimable). + * Thread-safe: all reads/writes are synchronized on the entry itself. The entry self-transitions to weak-ref + * state when its future completes successfully, via a callback registered at construction time. + * + *

Safety invariant: {@code CompletionStageFutureImpl} (extends {@code CompletableFuture}) retains its result + * strongly in an internal field even after listeners fire. This means any thread holding a reference to the future + * can still retrieve the class via {@code get()}, even after this entry has transitioned to weak-ref state. + * The weak reference only governs the cache's retention — not in-flight callers'.

+ */ + private static final class CacheEntry { + private CompletionStageFuture> future; + private WeakReference> classRef; + + CacheEntry(@NotNull CompletionStageFuture> future, + @NotNull String key, + @NotNull ReferenceQueue> queue) { + this.future = future; + // Self-transition: when compilation completes, swap to keyed weak ref + future.whenComplete((clazz, err) -> { + if (clazz != null) { + complete(clazz, key, queue); + } + // On failure, the entry stays in-flight with a failed future. + // The caller will remove it from the cache. + }); + } + + /** Transition to weak-ref state: releases the strong reference held by the completed future. */ + private synchronized void complete( + @NotNull Class clazz, @NotNull String key, @NotNull ReferenceQueue> queue) { + this.classRef = new KeyedWeakReference(clazz, key, queue); + this.future = null; + } + + /** + * Atomically resolve this entry's state. Returns one of: + *
    + *
  • A {@code Class} — cache hit, class still alive
    • + *
  • A {@code CompletionStageFuture>} — compilation in flight, wait on it
    • + *
  • {@code null} — entry is stale (class was GC'd), caller should retry
    • + *
+ */ + synchronized Optional>> resolve() { + if (classRef != null) { + final Class c = classRef.get(); + return Optional.ofNullable(c).map(CompletableFuture::completedFuture); + } + return Optional.of(future); + } + + /** True if completed but the class has been GC'd (stale). Used inside compute() lambda. */ + synchronized boolean isStale() { + return future == null && (classRef == null || classRef.get() == null); + } + } /** The context classloader captured at construction time; used as fallback parent for per-batch classloaders. */ private final ClassLoader parentClassLoader; @@ -222,6 +337,9 @@ public void compile( throw new IllegalArgumentException("Requests and resolvers must be the same length"); } + // Opportunistically clean up stale cache entries whose classes have been GC'd + evictStaleEntries(); + final boolean shouldTrace = log.isTraceEnabled() && Arrays.stream(requests).map(QueryCompilerRequest::packageNameRoot) .anyMatch(QueryCompilerImpl::shouldTrace); @@ -238,76 +356,80 @@ public void compile( }, requests).endl(); } - // Compute content hashes for deduplication - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } - final String[] hashes = new String[requests.length]; - for (int ii = 0; ii < requests.length; ++ii) { - hashes[ii] = ByteUtils.byteArrToHex(digest.digest( - requests[ii].classBody().getBytes(StandardCharsets.UTF_8))); - } - - // noinspection unchecked - final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; - + // For each request: cache hit, in-flight wait, or needs compilation final List newRequests = new ArrayList<>(); final List>> newResolvers = new ArrayList<>(); final List newHashes = new ArrayList<>(); + final List>> allFutures = new ArrayList<>(requests.length); - synchronized (this) { - for (int ii = 0; ii < requests.length; ++ii) { - final QueryCompilerRequest request = requests[ii]; - final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + for (int ii = 0; ii < requests.length; ++ii) { + // Compute content hash for deduplication + QueryCompilerRequest req = requests[ii]; + String hash = Hashing.sha256().hashString(req.classBody(), StandardCharsets.UTF_8).toString(); + + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + // Resolve cache state, retrying if we hit a stale entry race + Optional>> resolved; + while (true) { + final CacheEntry entry = cache.compute(hash, (key, existing) -> { + if (existing != null && !existing.isStale()) { + return existing; + } - CompletionStageFuture> future = - knownClasses.putIfAbsent(hashes[ii], resolver.getFuture()); - if (future == null) { - newRequests.add(request); + // No/stale existing entry, we will do the compilation ourselves newResolvers.add(resolver); - newHashes.add(hashes[ii]); - future = resolver.getFuture(); - } else if (shouldTrace) { - log.trace().append("Found existing future in knownClasses for ").append(request.className()) - .append(" (done=").append(future.isDone()).append(")").endl(); + newRequests.add(req); + newHashes.add(hash); + return new CacheEntry(resolver.getFuture(), key, staleQueue); + }); + + resolved = entry.resolve(); + if (resolved.isPresent()) { + // Got a future, either to our work, or someone else's + allFutures.add(resolved.get()); + break; } - allFutures[ii] = future; + // Stale race: entry transitioned between compute() and resolve(). Remove and retry. + cache.remove(hash, entry); } } + // Compile new requests if (!newResolvers.isEmpty()) { - // It's my job to fulfill these futures. try { compileHelper(newRequests, newResolvers, newHashes); } catch (RuntimeException e) { - // These failures are not applicable to a single request, so we can't just complete the future and - // leave the failure in the cache. - synchronized (this) { - for (int ii = 0; ii < newRequests.size(); ++ii) { - if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newHashes.get(ii)); - } + for (int ii = 0; ii < newHashes.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + cache.remove(newHashes.get(ii)); } } throw e; } + + // Remove cache entries for failed or incomplete compilations + // (successful ones self-transitioned to weak refs via whenComplete) + for (int ii = 0; ii < newHashes.size(); ++ii) { + try { + newResolvers.get(ii).getFuture().get(); // throws if completed exceptionally + } catch (Exception ex) { + cache.remove(newHashes.get(ii)); + } + } } + // Wait on in-flight futures from other compilations for (int ii = 0; ii < requests.length; ++ii) { + final Future> future = allFutures.get(ii); try { - final CompletionStageFuture> future = allFutures[ii]; - // TODO do we want a timeout here, knowing that this might leave other bad state? resolvers[ii].complete(future.get(10, TimeUnit.SECONDS)); } catch (TimeoutException err) { final String msg = "Timed out (10s) waiting for class compilation" + " request[" + ii + "] className=" + requests[ii].className() - + " future=" + allFutures[ii] - + " isDone=" + allFutures[ii].isDone(); + + " future=" + future + + " isDone=" + future.isDone(); log.error().append(msg).endl(); - // Fail all remaining resolvers and throw immediately final UncheckedDeephavenException timeout = new UncheckedDeephavenException(msg); for (int jj = ii; jj < requests.length; ++jj) { resolvers[jj].completeExceptionally(timeout); @@ -318,7 +440,7 @@ public void compile( } catch (InterruptedException err) { // This can only occur if we are interrupted while waiting for the future to complete from another // compilation request. - Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures.get(ii), "allFutures.get(ii)"); resolvers[ii].completeExceptionally(err); } catch (Throwable err) { resolvers[ii].completeExceptionally(err); @@ -477,7 +599,6 @@ private boolean doCompileAndDefineSingleRound( final MutableInt numFailures = new MutableInt(0); final List globalFailures = new ArrayList<>(); - compiler.getTask(compilerOutput, outputFm, diagnostic -> { From ddd3add1f8d0bc079433687ac561916fec018a77 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 30 Jun 2026 15:29:54 -0500 Subject: [PATCH 08/12] Spotless --- .../engine/context/QueryCompilerImpl.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index e319f9a3fcb..76c7985f0c7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -192,8 +192,8 @@ static QueryCompilerImpl createForUnitTests(final List classNamesForAnno /** * Cache from SHA-256 hash of class body to entry. While compilation is in-flight, the entry holds a future for * coordination. Once complete, the entry transitions to a weak reference so the class (and its classloader and - * bytecode) can be GC'd when no longer in use. All transitions are atomic via per-entry synchronization. - * Stale entries are cleaned up via a {@link ReferenceQueue} drained at the start of each {@link #compile} call. + * bytecode) can be GC'd when no longer in use. All transitions are atomic via per-entry synchronization. Stale + * entries are cleaned up via a {@link ReferenceQueue} drained at the start of each {@link #compile} call. */ private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); @@ -223,13 +223,15 @@ private static final class KeyedWeakReference extends WeakReference> { /** * A cache entry that transitions from in-flight (future-based coordination) to weakly-cached (reclaimable). - * Thread-safe: all reads/writes are synchronized on the entry itself. The entry self-transitions to weak-ref - * state when its future completes successfully, via a callback registered at construction time. + * Thread-safe: all reads/writes are synchronized on the entry itself. The entry self-transitions to weak-ref state + * when its future completes successfully, via a callback registered at construction time. * - *

Safety invariant: {@code CompletionStageFutureImpl} (extends {@code CompletableFuture}) retains its result + *

+ * Safety invariant: {@code CompletionStageFutureImpl} (extends {@code CompletableFuture}) retains its result * strongly in an internal field even after listeners fire. This means any thread holding a reference to the future - * can still retrieve the class via {@code get()}, even after this entry has transitioned to weak-ref state. - * The weak reference only governs the cache's retention — not in-flight callers'.

+ * can still retrieve the class via {@code get()}, even after this entry has transitioned to weak-ref state. The + * weak reference only governs the cache's retention — not in-flight callers'. + *

*/ private static final class CacheEntry { private CompletionStageFuture> future; @@ -259,9 +261,9 @@ private synchronized void complete( /** * Atomically resolve this entry's state. Returns one of: *
    - *
  • A {@code Class} — cache hit, class still alive
    • - *
  • A {@code CompletionStageFuture>} — compilation in flight, wait on it
    • - *
  • {@code null} — entry is stale (class was GC'd), caller should retry
    • + *
  • A {@code Class} — cache hit, class still alive
    • + *
  • A {@code CompletionStageFuture>} — compilation in flight, wait on it
    • + *
  • {@code null} — entry is stale (class was GC'd), caller should retry
    • *
*/ synchronized Optional>> resolve() { @@ -541,7 +543,7 @@ private void compileAndDefine(@NotNull final List req try { // TODO Probably should be configurable, probably shouldn't be indefinite? - // Allowing timeout like this may require cleanup + // Allowing timeout like this may require cleanup final boolean completed = latch.await(10, TimeUnit.SECONDS); if (!completed) { final String msg = "QueryCompilerImpl.compileAndDefine: latch timed out after 10s!" From 7fd6f32c4defe4c197953fe904b8944f8e84a0a2 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 1 Jul 2026 07:25:18 -0500 Subject: [PATCH 09/12] Review cleanup: * restore private access to method * infinite wait on compilation --- .../engine/context/QueryCompilerImpl.java | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 76c7985f0c7..e65339600ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -62,8 +62,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.jar.Attributes; @@ -425,18 +423,7 @@ public void compile( for (int ii = 0; ii < requests.length; ++ii) { final Future> future = allFutures.get(ii); try { - resolvers[ii].complete(future.get(10, TimeUnit.SECONDS)); - } catch (TimeoutException err) { - final String msg = "Timed out (10s) waiting for class compilation" - + " request[" + ii + "] className=" + requests[ii].className() - + " future=" + future - + " isDone=" + future.isDone(); - log.error().append(msg).endl(); - final UncheckedDeephavenException timeout = new UncheckedDeephavenException(msg); - for (int jj = ii; jj < requests.length; ++jj) { - resolvers[jj].completeExceptionally(timeout); - } - throw timeout; + resolvers[ii].complete(future.get()); } catch (ExecutionException err) { resolvers[ii].completeExceptionally(err.getCause()); } catch (InterruptedException err) { @@ -542,15 +529,7 @@ private void compileAndDefine(@NotNull final List req onError); try { - // TODO Probably should be configurable, probably shouldn't be indefinite? - // Allowing timeout like this may require cleanup - final boolean completed = latch.await(10, TimeUnit.SECONDS); - if (!completed) { - final String msg = "QueryCompilerImpl.compileAndDefine: latch timed out after 10s!" - + " numTasks=" + numTasks + " requests=" + requests.size(); - log.error().append(msg).endl(); - throw new UncheckedDeephavenException(msg); - } + latch.await(); final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); if (perfEntry != null) { QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); @@ -876,7 +855,7 @@ private static String loadIdentifyingField(Class c) { } } - static String makeFinalCode(String className, String classBody, String packageName) { + private static String makeFinalCode(String className, String classBody, String packageName) { if (classBody.contains("$CLASSNAME$")) { throw new IllegalArgumentException("QueryCompiler's support of the $CLASSNAME$ variable has been removed as" + " the final class name affects the compiled byte code and therefore cannot be dynamically " From 24b6cfabd0a5ce2157488f7ad01b633616e671e0 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 2 Jul 2026 10:38:10 -0500 Subject: [PATCH 10/12] Updated docs/examples, removed silly test, restored backward compat --- docs/groovy/conceptual/execution-context.md | 6 +-- docs/python/conceptual/execution-context.md | 12 ++--- .../engine/context/QueryCompiler.java | 6 +-- .../engine/context/TestQueryCompiler.java | 10 +--- .../engine/context/QueryCompilerImpl.java | 54 +++++++++++-------- .../impl/TestEventDrivenUpdateGraph.java | 2 +- .../engine/context/TestExecutionContext.java | 2 +- .../parquet/table/TableWriteBenchmark.java | 2 +- .../impl/util/TestUpdateAncestorViz.java | 2 +- 9 files changed, 46 insertions(+), 50 deletions(-) diff --git a/docs/groovy/conceptual/execution-context.md b/docs/groovy/conceptual/execution-context.md index c658579bc7d..ca537fa88a3 100644 --- a/docs/groovy/conceptual/execution-context.md +++ b/docs/groovy/conceptual/execution-context.md @@ -235,7 +235,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile())) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ``` @@ -248,7 +248,7 @@ This approach allows you to specify: - **Query scope**: A new, empty query scope via `newQueryScope`. - **Operation initializer**: Parallelization behavior of operations. - **Update graph**: A custom update graph (e.g., `PeriodicUpdateGraph` or `EventDrivenUpdateGraph`). -- **Query compiler**: A compiler instance with a specified working directory and class loader. +- **Query compiler**: A compiler instance with an optional directory to load classes from and classloader. For use cases requiring event-driven updates instead of periodic updates, you can substitute an `EventDrivenUpdateGraph`: @@ -262,7 +262,7 @@ executionContext = ExecutionContext.newBuilder() .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(eventDrivenGraph) - .setQueryCompiler(QueryCompilerImpl.create(Files.createTempDirectory("qc_").toFile())) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ``` diff --git a/docs/python/conceptual/execution-context.md b/docs/python/conceptual/execution-context.md index 23fbbdf6edd..6a53c77f350 100644 --- a/docs/python/conceptual/execution-context.md +++ b/docs/python/conceptual/execution-context.md @@ -260,11 +260,7 @@ execution_context = ( .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(PeriodicUpdateGraph.newBuilder("MyCustomGraph").build()) - .setQueryCompiler( - QueryCompilerImpl.create( - jpy.get_type("java.io.File")(temp_dir), - ) - ) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ) ``` @@ -278,7 +274,7 @@ This approach allows you to specify: - **Query scope**: A new, empty query scope via `newQueryScope`. - **Operation initializer**: Parallelization behavior of operations. - **Update graph**: A custom update graph (e.g., `PeriodicUpdateGraph` or `EventDrivenUpdateGraph`). -- **Query compiler**: A compiler instance with a specified working directory and class loader. +- **Query compiler**: A custom query compiler, which can be created with an optional class directory, plus optional classloader For use cases requiring event-driven updates instead of periodic updates, you can substitute an `EventDrivenUpdateGraph`: @@ -306,9 +302,7 @@ execution_context = ( .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(event_driven_graph) .setQueryCompiler( - QueryCompilerImpl.create( - jpy.get_type("java.io.File")(temp_dir), - ) + QueryCompilerImpl.create() ) .build() ) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index f9adfc895a4..638946bb13e 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -12,12 +12,12 @@ /** * Java compiler api, specifically geared towards query language expressions. Implementations should be constructed with - * a path on disk to existing bytecode to use in addition to `java.class.path`. Likewise, callers must ensure that the - * context classloader already contains those input classes - the new class will be loaded in a child classloader to + * a path on disk to existing bytecode to use in addition to {@code java.class.path}. Likewise, callers must ensure that + * the context classloader already contains those input classes - the new class will be loaded in a child classloader to * ensure that the class can be GCd when neither it nor the compiler instance are reachable. *

* Requests for compilation will never depend on other classes compiled by this or any other QueryCompiler, and are - * expected to be unique based on the provided classname. This enables the implementation to cache classes and only + * expected to be unique based on the provided class name. This enables the implementation to cache classes and only * compile and load a class once per name. */ public interface QueryCompiler { diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index fd5c1ff5ba1..7b6459473b8 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -65,7 +65,7 @@ public void setUp() { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.create()) .build() .open(); } @@ -368,14 +368,6 @@ public void testBadCompile() { (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { CompletionStageFuture.make(), }; - - final QueryCompilerImpl badCompiler = - QueryCompilerImpl.createForUnitTests(List.of("InvalidClassArgument")); - UncheckedDeephavenException e = org.junit.Assert.assertThrows(UncheckedDeephavenException.class, - () -> badCompiler.compile(requests, resolvers)); - org.junit.Assert.assertEquals("Error Invoking Compiler, no source present in diagnostic:\n" + - "Class names, 'InvalidClassArgument', are only accepted if annotation processing is explicitly requested", - e.getMessage()); } @Test diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index e65339600ab..735cf854578 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -76,8 +76,8 @@ *

* The compiler resolves dependencies from {@code java.class.path} and an optional additional classpath directory * (typically where Groovy writes its bytecode). Compiled classes are loaded in per-batch child classloaders of the - * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Class nor this - * compiler instance are reachable. Returned classes then + * provided parent classloader, enabling GC of both classes and classloaders when neither the compiled Classes nor this + * compiler instance are reachable. */ public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { @@ -173,18 +173,25 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) */ public static QueryCompiler create(@Nullable final File additionalClassPathDir) { - return new QueryCompilerImpl(additionalClassPathDir, null); + return create(additionalClassPathDir, null); } /** - * Creates a new compiler that has no extra directory to read from, suitable for unit tests. + * Creates a new QueryCompilerImpl. Uses the current thread's context classloader as the parent for per-batch + * classloaders (as required by {@link QueryCompiler}'s contract). + * + * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) */ - public static QueryCompiler createForUnitTests() { - return createForUnitTests(null); + public static QueryCompiler create(@Nullable final File additionalClassPathDir, @Nullable ClassLoader classLoader) { + return new QueryCompilerImpl(additionalClassPathDir, classLoader); } - static QueryCompilerImpl createForUnitTests(final List classNamesForAnnotationProcessing) { - return new QueryCompilerImpl(null, classNamesForAnnotationProcessing); + /** + * Creates a new compiler that has no extra directory to read from, suitable for unit tests or cases where the + * existing classpath is sufficient. + */ + public static QueryCompiler create() { + return create(null, null); } /** @@ -278,23 +285,20 @@ synchronized boolean isStale() { } } - /** The context classloader captured at construction time; used as fallback parent for per-batch classloaders. */ + /** The context classloader provided at creation time, or null if none set */ + @Nullable private final ClassLoader parentClassLoader; /** Optional additional classpath directory (e.g., where Groovy writes bytecode). */ @Nullable private final File additionalClassPathDir; - // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. - private final List classNamesForAnnotationProcessing; - private QueryCompilerImpl( @Nullable final File additionalClassPathDir, - final List classNamesForAnnotationProcessing) { + @Nullable final ClassLoader parentClassLoader) { ensureJavaCompiler(); this.additionalClassPathDir = additionalClassPathDir; - this.parentClassLoader = Thread.currentThread().getContextClassLoader(); - this.classNamesForAnnotationProcessing = classNamesForAnnotationProcessing; + this.parentClassLoader = parentClassLoader; if (log.isTraceEnabled()) { log.trace().append("QueryCompiler Class Path: ").append(getClassPath()).append(File.pathSeparator) @@ -606,7 +610,7 @@ private boolean doCompileAndDefineSingleRound( } }, compilerOptions, - classNamesForAnnotationProcessing, + null, requests.subList(startInclusive, endExclusive).stream() .map(CompilationRequestAttempt::makeSource) .collect(Collectors.toList())) @@ -634,10 +638,10 @@ private boolean doCompileAndDefineSingleRound( .append(", range=[").append(startInclusive).append(",").append(endExclusive).append(")") .endl(); if (!compiledClasses.isEmpty()) { - // Use the current thread's context classloader as parent so that Groovy-defined classes - // (which may have been loaded after this QueryCompilerImpl was constructed) are visible. - final ClassLoader currentContextCl = Thread.currentThread().getContextClassLoader(); - final ClassLoader batchParent = currentContextCl != null ? currentContextCl : parentClassLoader; + // Use the current thread's context classloader as parent, unless we were explicitly provided a different + // one + final ClassLoader batchParent = + parentClassLoader != null ? parentClassLoader : Thread.currentThread().getContextClassLoader();; final BatchClassLoader batchCl = new BatchClassLoader(batchParent, compiledClasses); for (final CompilationRequestAttempt request : requests.subList(startInclusive, endExclusive)) { @@ -653,7 +657,7 @@ private boolean doCompileAndDefineSingleRound( continue; } - // Load the top-level class (which triggers loading of inner/anonymous classes as needed) + // Load the outer class for this request final Class clazz; try { clazz = batchCl.loadClass(request.fqClassName); @@ -750,8 +754,14 @@ Map getCompiledClasses() { @Override public JavaFileObject getJavaFileForOutput( Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { + if (location != StandardLocation.CLASS_OUTPUT || kind != JavaFileObject.Kind.CLASS) { + throw new IllegalArgumentException("In-memory output file manager only supports writing bytecode"); + } final InMemoryClassFileObject fileObject = new InMemoryClassFileObject(className); - outputClasses.put(className, fileObject); + InMemoryClassFileObject existing = outputClasses.put(className, fileObject); + if (existing != null) { + throw new IllegalArgumentException("Attempted to write duplicate class name " + className); + } return fileObject; } diff --git a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java index b1c667e25f8..08b0e0439f9 100644 --- a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java +++ b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java @@ -112,7 +112,7 @@ public void run() { } private QueryCompiler compilerForUnitTests() { - return QueryCompilerImpl.createForUnitTests(); + return QueryCompilerImpl.create(); } @Test diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index da1d636c03b..10435309133 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.create()) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index b38c4a78ed5..dd10bb8c06e 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompilerImpl.createForUnitTests()) + QueryCompilerImpl.create()) .build(); exContextCloseable = context.open(); diff --git a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java index d8085d8c409..c3ceb18b917 100644 --- a/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java +++ b/extensions/performance/src/test/java/io/deephaven/engine/table/impl/util/TestUpdateAncestorViz.java @@ -56,7 +56,7 @@ public void before() throws IOException { cacheDir.deleteOnExit(); executionContext = ExecutionContext.newBuilder().newQueryLibrary().newQueryScope() - .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.create()) .setOperationInitializer(ForkJoinPoolOperationInitializer.fromCommonPool()) .setUpdateGraph(defaultUpdateGraph).build().withAuthContext(new AuthContext.Anonymous()); } From 6f7153cf7165c6a70e726aca94d6ff4f7c91fd7a Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 7 Jul 2026 10:21:35 -0500 Subject: [PATCH 11/12] Restore bad compile test --- .../engine/context/TestQueryCompiler.java | 10 +++++ .../engine/context/QueryCompilerImpl.java | 39 +++++++++++++------ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 7b6459473b8..11af4585216 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -368,6 +368,16 @@ public void testBadCompile() { (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { CompletionStageFuture.make(), }; + + final QueryCompilerImpl badCompiler = QueryCompilerImpl.create(); + badCompiler.setClassNamesForAnnotationProcessing(List.of("InvalidClassArgument")); + + UncheckedDeephavenException e = org.junit.Assert.assertThrows(UncheckedDeephavenException.class, + () -> badCompiler.compile(requests, resolvers)); + org.junit.Assert.assertEquals("Error Invoking Compiler, no source present in diagnostic:\n" + + "Class names, 'InvalidClassArgument', are only accepted if annotation processing is explicitly requested", + e.getMessage()); + } @Test diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 735cf854578..68ad3a58899 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -20,6 +20,7 @@ import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; @@ -127,7 +128,6 @@ private static boolean shouldTrace(String className) { return TRACE_INCLUDE_PREFIXES.stream().anyMatch(className::startsWith); } - private static JavaCompiler compiler; private static final AtomicReference fileManagerCache = new AtomicReference<>(); @@ -172,7 +172,7 @@ private static void releaseFileManager(@NotNull final JavaFileManager fileManage * * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) */ - public static QueryCompiler create(@Nullable final File additionalClassPathDir) { + public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir) { return create(additionalClassPathDir, null); } @@ -182,7 +182,8 @@ public static QueryCompiler create(@Nullable final File additionalClassPathDir) * * @param additionalClassPathDir optional directory to add to the compiler's classpath (e.g., groovy bytecode dir) */ - public static QueryCompiler create(@Nullable final File additionalClassPathDir, @Nullable ClassLoader classLoader) { + public static QueryCompilerImpl create(@Nullable final File additionalClassPathDir, + @Nullable ClassLoader classLoader) { return new QueryCompilerImpl(additionalClassPathDir, classLoader); } @@ -190,7 +191,7 @@ public static QueryCompiler create(@Nullable final File additionalClassPathDir, * Creates a new compiler that has no extra directory to read from, suitable for unit tests or cases where the * existing classpath is sufficient. */ - public static QueryCompiler create() { + public static QueryCompilerImpl create() { return create(null, null); } @@ -205,13 +206,8 @@ public static QueryCompiler create() { /** Queue that receives cleared weak references, allowing us to remove stale cache entries. */ private final ReferenceQueue> staleQueue = new ReferenceQueue<>(); - /** Drains the stale queue and removes any cache entries whose weak references have been cleared by GC. */ - private void evictStaleEntries() { - KeyedWeakReference ref; - while ((ref = (KeyedWeakReference) staleQueue.poll()) != null) { - cache.remove(ref.key); - } - } + // This is for test use only, specifying a non-null list causes an error without a specific source to be generated. + private List classNamesForAnnotationProcessing; /** * A WeakReference that remembers its cache key, so we can remove the entry when the referent is GC'd. @@ -306,6 +302,25 @@ private QueryCompilerImpl( } } + /** Drains the stale queue and removes any cache entries whose weak references have been cleared by GC. */ + private void evictStaleEntries() { + KeyedWeakReference ref; + while ((ref = (KeyedWeakReference) staleQueue.poll()) != null) { + cache.remove(ref.key); + } + } + + /** + * Currently only intended for testing, this method allows specifying annotation processors to run during this + * compile. + * + * @param classNames the annotation processor classes to allow to run + */ + @TestUseOnly + public void setClassNamesForAnnotationProcessing(List classNames) { + classNamesForAnnotationProcessing = classNames; + } + @Override public LogOutput append(LogOutput logOutput) { return logOutput.append("QueryCompilerImpl{additionalClassPathDir=") @@ -610,7 +625,7 @@ private boolean doCompileAndDefineSingleRound( } }, compilerOptions, - null, + classNamesForAnnotationProcessing, requests.subList(startInclusive, endExclusive).stream() .map(CompilationRequestAttempt::makeSource) .collect(Collectors.toList())) From 0ce91055dcda5d139a05e00747f01270cb75664f Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 7 Jul 2026 15:05:15 -0500 Subject: [PATCH 12/12] docs format --- docs/python/conceptual/execution-context.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/python/conceptual/execution-context.md b/docs/python/conceptual/execution-context.md index 6a53c77f350..ef50996cda2 100644 --- a/docs/python/conceptual/execution-context.md +++ b/docs/python/conceptual/execution-context.md @@ -301,9 +301,7 @@ execution_context = ( .newQueryScope() .setOperationInitializer(OperationInitializer.NON_PARALLELIZABLE) .setUpdateGraph(event_driven_graph) - .setQueryCompiler( - QueryCompilerImpl.create() - ) + .setQueryCompiler(QueryCompilerImpl.create()) .build() ) ```