Handle layered and Darwin AWT JNI registration

Reapply the AWT setup when awt is inherited from a previous layer, tighten the desktop integration tests once environment checks pass, and keep Darwin absolute built-in AWT loads restricted to trusted image and JDK library locations.

Author: thomaswue

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/NativeLibraries.java

@@ -32,13 +32,13 @@
 import java.util.Collection;
 
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.nativeimage.impl.ProcessPropertiesSupport;
 import org.graalvm.word.PointerBase;
 import org.graalvm.word.impl.Word;
 
 import com.oracle.svm.core.NeverInline;
+import com.oracle.svm.core.OS;
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.shared.util.StringUtil;
@@ -103,6 +103,9 @@ public void loadLibraryAbsolute(File file) {
         if (loadLibrary0(file, false)) {
             return;
         }
+        if (loadBuiltinDarwinLibraryAbsoluteFallback(file)) {
+            return;
+        }
         throw new UnsatisfiedLinkError("Can't load library: " + file);
     }
 
@@ -145,12 +148,6 @@ public void loadLibraryRelative(String name) {
 
     private boolean loadLibrary0(File file, boolean builtin) {
         try {
-            if (!builtin) {
-                String builtInName = asBuiltinLibraryName(file.getName());
-                if (builtInName != null && addLibrary(builtInName, true)) {
-                    return true;
-                }
-            }
             String canonical = builtin ? file.getName() : file.getCanonicalPath();
             return addLibrary(canonical, builtin);
         } catch (IOException e) {
@@ -159,12 +156,47 @@ private boolean loadLibrary0(File file, boolean builtin) {
     }
 
     private static String asBuiltinLibraryName(String fileName) {
-        if (Platform.includedIn(Platform.DARWIN.class) && fileName.startsWith("lib") && fileName.endsWith(".dylib")) {
+        if (OS.getCurrent() == OS.DARWIN && fileName.startsWith("lib") && fileName.endsWith(".dylib")) {
             return fileName.substring("lib".length(), fileName.length() - ".dylib".length());
         }
         return null;
     }
 
+    private boolean loadBuiltinDarwinLibraryAbsoluteFallback(File file) {
+        String builtInName = asBuiltinLibraryName(file.getName());
+        if (builtInName == null) {
+            return false;
+        }
+        try {
+            return isBuiltinDarwinLibraryLocation(file) && addLibrary(builtInName, true);
+        } catch (IOException e) {
+            return false;
+        }
+    }
+
+    private static boolean isBuiltinDarwinLibraryLocation(File file) throws IOException {
+        if (OS.getCurrent() != OS.DARWIN) {
+            return false;
+        }
+
+        File canonicalParent = file.getCanonicalFile().getParentFile();
+        if (canonicalParent == null) {
+            return false;
+        }
+
+        String imageDirectory = getImageDirectory();
+        if (imageDirectory != null && canonicalParent.equals(new File(imageDirectory).getCanonicalFile())) {
+            return true;
+        }
+
+        String javaHome = System.getProperty("java.home");
+        if (javaHome != null && canonicalParent.equals(new File(javaHome, "lib").getCanonicalFile())) {
+            return true;
+        }
+
+        return false;
+    }
+
     protected abstract boolean addLibrary(String canonical, boolean builtin);
 
     public abstract PointerBase findSymbol(String name);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterSupport.java

@@ -134,7 +134,7 @@ private static boolean recommendTraceAgentForAWT() {
         if (!ImageSingletons.contains(JNIRegistrationSupport.class) || !ImageSingletons.contains(JNIReflectionDictionary.class)) {
             return false;
         }
-        if (!JNIRegistrationSupport.singleton().isRegisteredLibrary("awt")) {
+        if (!JNIRegistrationSupport.singleton().isAnyLayerRegisteredLibrary("awt")) {
             return false; // AWT not used
         }
         // check if any class located in java.awt or sun.awt is registered for JNI access

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationAWTSupport.java

@@ -62,7 +62,11 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
     @Override
     public void afterAnalysis(AfterAnalysisAccess access) {
         JNIRegistrationSupport jniRegistrationSupport = JNIRegistrationSupport.singleton();
-        if (jniRegistrationSupport.isRegisteredLibrary("awt")) {
+        if ((isLinux() || isDarwin()) && jniRegistrationSupport.isPreviousLayerRegisteredLibrary("awt") &&
+                        !jniRegistrationSupport.isCurrentLayerRegisteredLibrary("awt")) {
+            registerHeadlessJavaDesktopSupport("awt");
+        }
+        if (jniRegistrationSupport.isAnyLayerRegisteredLibrary("awt")) {
             jniRegistrationSupport.addJvmShimExports(
                             "JVM_IsStaticallyLinked");
             jniRegistrationSupport.addJavaShimExports(
@@ -104,11 +108,11 @@ public void afterAnalysis(AfterAnalysisAccess access) {
                 jniRegistrationSupport.registerLibrary("awt_xawt");
             }
         }
-        if (jniRegistrationSupport.isRegisteredLibrary("javaaccessbridge")) {
+        if (jniRegistrationSupport.isAnyLayerRegisteredLibrary("javaaccessbridge")) {
             /* Dependency on `jawt` is not expressed in Java, so we register it manually here. */
             jniRegistrationSupport.registerLibrary("jawt");
         }
-        if (jniRegistrationSupport.isRegisteredLibrary("javajpeg")) {
+        if (jniRegistrationSupport.isAnyLayerRegisteredLibrary("javajpeg")) {
             jniRegistrationSupport.addJavaShimExports(
                             "JNU_GetEnv",
                             "JNU_ThrowByName",
@@ -119,7 +123,7 @@ public void afterAnalysis(AfterAnalysisAccess access) {
 
     @Override
     public void beforeImageWrite(BeforeImageWriteAccess access) {
-        if (isDarwin() && JNIRegistrationSupport.singleton().isRegisteredLibrary("awt")) {
+        if (isDarwin() && JNIRegistrationSupport.singleton().isAnyLayerRegisteredLibrary("awt")) {
             ((BeforeImageWriteAccessImpl) access).registerLinkerInvocationTransformer(linkerInvocation -> {
                 linkerInvocation.addNativeLinkerOption("-Wl,-framework,AppKit");
                 linkerInvocation.addNativeLinkerOption("-Wl,-framework,Accelerate");
@@ -134,7 +138,7 @@ public void beforeImageWrite(BeforeImageWriteAccess access) {
                 return linkerInvocation;
             });
         }
-        if (isWindows() && JNIRegistrationSupport.singleton().isRegisteredLibrary("awt")) {
+        if (isWindows() && JNIRegistrationSupport.singleton().isAnyLayerRegisteredLibrary("awt")) {
             ((BeforeImageWriteAccessImpl) access).registerLinkerInvocationTransformer(linkerInvocation -> {
                 /*
                  * Add Windows libraries that are pulled in as a side effect of exporting the

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationSupport.java

@@ -188,10 +188,18 @@ private void addLibrary(String libname) {
         }
     }
 
-    public boolean isRegisteredLibrary(String libname) {
+    boolean isCurrentLayerRegisteredLibrary(String libname) {
         return jniRegistrationSupportSingleton.currentLayerRegisteredLibraries.contains(libname);
     }
 
+    boolean isPreviousLayerRegisteredLibrary(String libname) {
+        return jniRegistrationSupportSingleton.prevLayerRegisteredLibraries.contains(libname);
+    }
+
+    public boolean isAnyLayerRegisteredLibrary(String libname) {
+        return isCurrentLayerRegisteredLibrary(libname) || isPreviousLayerRegisteredLibrary(libname);
+    }
+
     /** Adds exports that `jvm` shim should re-export. */
     void addJvmShimExports(String... exports) {
         addShimExports("jvm", exports);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jni/DarwinBuiltinJNISymbolSupport.java

@@ -117,7 +117,7 @@ private static Set<String> scanAvailableSymbols() throws IOException, Interrupte
     }
 
     private static Set<String> readDefinedSymbols(Path staticLibrary) throws IOException, InterruptedException {
-        List<String> commandLine = List.of("nm", "-g", "-U", "-j", staticLibrary.toString());
+        List<String> commandLine = List.of("nm", "--extern-only", "--defined-only", "--format=just-symbols", staticLibrary.toString());
         ProcessBuilder command = FileUtils.prepareCommand(commandLine, null).redirectErrorStream(true);
         FileUtils.traceCommand(command);
         Process process = command.start();

substratevm/src/com.oracle.svm.test/src/com/oracle/svm/integrationtest/HeadlessJavaDesktopTest.java

@@ -39,7 +39,7 @@ public class HeadlessJavaDesktopTest {
     @Test
     public void headlessJavaDesktopSmokeTest() throws Exception {
         String osName = System.getProperty("os.name", "");
-        Assume.assumeTrue(osName.startsWith("Linux") || osName.startsWith("Mac") || osName.startsWith("Darwin"));
+        Assume.assumeTrue(osName.startsWith("Linux") || osName.startsWith("Mac"));
 
         System.setProperty("java.awt.headless", "true");
 

substratevm/src/com.oracle.svm.test/src/com/oracle/svm/integrationtest/NonHeadlessJavaDesktopTest.java

@@ -24,10 +24,8 @@
  */
 package com.oracle.svm.integrationtest;
 
-import java.awt.AWTError;
 import java.awt.Dimension;
 import java.awt.GraphicsEnvironment;
-import java.awt.HeadlessException;
 import java.awt.Toolkit;
 import java.awt.image.BufferedImage;
 import java.io.ByteArrayOutputStream;
@@ -48,16 +46,9 @@ public void nonHeadlessJavaDesktopSmokeTest() throws Exception {
         Assume.assumeTrue(display != null && !display.isEmpty());
 
         System.clearProperty("java.awt.headless");
-        Toolkit toolkit;
-        Dimension screenSize;
-        try {
-            Assume.assumeFalse(GraphicsEnvironment.isHeadless());
-            toolkit = Toolkit.getDefaultToolkit();
-            screenSize = toolkit.getScreenSize();
-        } catch (AWTError | HeadlessException e) {
-            Assume.assumeNoException(e);
-            return;
-        }
+        Assume.assumeFalse(GraphicsEnvironment.isHeadless());
+        Toolkit toolkit = Toolkit.getDefaultToolkit();
+        Dimension screenSize = toolkit.getScreenSize();
         Assert.assertTrue("Invalid screen width: " + screenSize.width, screenSize.width > 0);
         Assert.assertTrue("Invalid screen height: " + screenSize.height, screenSize.height > 0);