[GR-62402] Introduce source options

PullRequest: graalpython/4311

Author: msimacek

CHANGELOG.md

@@ -3,6 +3,11 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 25.2.0
+* Add support for [Truffle source options](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/source/Source.SourceBuilder.html#option(java.lang.String,java.lang.String)):
+  * The `python.Optimize` option can be used to specify the optimization level, like the `-O` (level 1) and `-OO` (level 2) commandline options.
+  * The `python.NewGlobals` option can be used to run a source with a fresh globals dictionary instead of the main module globals, which is useful for embeddings that want isolated top-level execution.
+
 ## Version 25.1.0
 * Intern string literals in source files
 * Allocation reporting via Truffle has been removed. Python object sizes were never reported correctly, so the data was misleading and there was a non-neglible overhead for object allocations even when reporting was inactive.

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java

@@ -82,6 +82,7 @@ public void before() {
         Builder newBuilder = Context.newBuilder();
         newBuilder.allowExperimentalOptions(true);
         newBuilder.allowAllAccess(true);
+        newBuilder.option("engine.WarnInterpreterOnly", "false");
         PythonTests.closeContext();
         tester = new DebuggerTester(newBuilder);
     }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/interop/SourceOptionsTests.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.interop;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.Source;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.oracle.graal.python.test.PythonTests;
+
+public class SourceOptionsTests extends PythonTests {
+    private Context context;
+
+    @Before
+    public void setUpTest() {
+        Context.Builder builder = Context.newBuilder();
+        builder.allowExperimentalOptions(true);
+        builder.allowAllAccess(true);
+        context = builder.build();
+    }
+
+    @After
+    public void tearDown() {
+        context.close();
+    }
+
+    @Test
+    public void testDefaultUsesMainModuleGlobals() {
+        context.eval("python", "x = 41");
+        assertEquals(42, context.eval("python", "x + 1").asInt());
+    }
+
+    @Test
+    public void testNewGlobalsIsolatedFromMainModule() throws IOException {
+        context.eval("python", "x = 41");
+
+        Source source = Source.newBuilder("python", "x = 100\nx + 1", "new-globals.py").option("python.NewGlobals", "true").build();
+
+        assertEquals(101, context.eval(source).asInt());
+        assertEquals(42, context.eval("python", "x + 1").asInt());
+    }
+
+    @Test
+    public void testSeparateNewGlobalsExecutionsDoNotShareState() throws IOException {
+        Source writeSource = Source.newBuilder("python", "x = 100", "new-globals-write.py").option("python.NewGlobals", "true").build();
+        Source readSource = Source.newBuilder("python", "x = globals().get('x', 0)\nx + 1", "new-globals-read.py").option("python.NewGlobals", "true").build();
+
+        context.eval(writeSource);
+        assertEquals(1, context.eval(readSource).asInt());
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -39,11 +39,9 @@
 import java.lang.invoke.VarHandle;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.InvalidPathException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.WeakHashMap;
@@ -108,6 +106,7 @@
 import com.oracle.graal.python.runtime.PythonContext.PythonThreadState;
 import com.oracle.graal.python.runtime.PythonImageBuildOptions;
 import com.oracle.graal.python.runtime.PythonOptions;
+import com.oracle.graal.python.runtime.PythonSourceOptions;
 import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.graal.python.runtime.object.PFactory;
 import com.oracle.graal.python.util.Function;
@@ -155,12 +154,7 @@
                 sandbox = SandboxPolicy.UNTRUSTED, //
                 implementationName = PythonLanguage.IMPLEMENTATION_NAME, //
                 version = PythonLanguage.VERSION, //
-                characterMimeTypes = {PythonLanguage.MIME_TYPE,
-                                "text/x-python-\0\u0000-eval", "text/x-python-\0\u0000-compile", "text/x-python-\1\u0000-eval", "text/x-python-\1\u0000-compile", "text/x-python-\2\u0000-eval",
-                                "text/x-python-\2\u0000-compile", "text/x-python-\0\u0100-eval", "text/x-python-\0\u0100-compile", "text/x-python-\1\u0100-eval", "text/x-python-\1\u0100-compile",
-                                "text/x-python-\2\u0100-eval", "text/x-python-\2\u0100-compile", "text/x-python-\0\u0040-eval", "text/x-python-\0\u0040-compile", "text/x-python-\1\u0040-eval",
-                                "text/x-python-\1\u0040-compile", "text/x-python-\2\u0040-eval", "text/x-python-\2\u0040-compile", "text/x-python-\0\u0140-eval", "text/x-python-\0\u0140-compile",
-                                "text/x-python-\1\u0140-eval", "text/x-python-\1\u0140-compile", "text/x-python-\2\u0140-eval", "text/x-python-\2\u0140-compile"}, //
+                characterMimeTypes = {PythonLanguage.MIME_TYPE}, //
                 defaultMimeType = PythonLanguage.MIME_TYPE, //
                 dependentLanguages = {"nfi", "llvm"}, //
                 interactive = true, internal = false, //
@@ -274,48 +268,6 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
 
     public static final String MIME_TYPE = "text/x-python";
 
-    // the syntax for mime types is as follows
-    // <mime> ::= "text/x-python-" <optlevel> <flags> "-" kind
-    // <kind> ::= "compile" | "eval"
-    // <optlevel> ::= "\0" | "\1" | "\2"
-    // <flags> ::= "\u0040" | "\u0100" | "\u0140" | "\u0000"
-    // where 0100 implies annotations, and 0040 implies barry_as_flufl
-    static final String MIME_PREFIX = MIME_TYPE + "-";
-    static final int OPT_FLAGS_LEN = 2; // 1 char is optlevel, 1 char is flags
-    static final String MIME_KIND_COMPILE = "compile";
-    static final String MIME_KIND_EVAL = "eval";
-    // Since flags are greater than the highest unicode codepoint, we shift them into more
-    // reasonable values in the mime type. 4 hex digits
-    static final int MIME_FLAG_SHIFTBY = 4 * 4;
-    // a dash follows after the opt flag pair
-    static final int MIME_KIND_START = MIME_PREFIX.length() + OPT_FLAGS_LEN + 1;
-
-    private static boolean mimeTypesComplete(ArrayList<String> mimeJavaStrings) {
-        ArrayList<String> mimeTypes = new ArrayList<>();
-        FutureFeature[] all = FutureFeature.values();
-        for (int flagset = 0; flagset < (1 << all.length); ++flagset) {
-            int flags = 0;
-            for (int i = 0; i < all.length; ++i) {
-                if ((flagset & (1 << i)) != 0) {
-                    flags |= all[i].flagValue;
-                }
-            }
-            for (int opt = 0; opt <= 2; opt++) {
-                for (String typ : new String[]{MIME_KIND_EVAL, MIME_KIND_COMPILE}) {
-                    mimeTypes.add(MIME_PREFIX + optFlagsToMime(opt, flags) + "-" + typ);
-                    mimeJavaStrings.add(String.format("\"%s\\%d\\u%04x-%s\"", MIME_PREFIX, opt, flags >> MIME_FLAG_SHIFTBY, typ));
-                }
-            }
-        }
-        HashSet<String> currentMimeTypes = new HashSet<>(List.of(PythonLanguage.class.getAnnotation(Registration.class).characterMimeTypes()));
-        return currentMimeTypes.containsAll(mimeTypes);
-    }
-
-    static {
-        ArrayList<String> mimeJavaStrings = new ArrayList<>();
-        assert mimeTypesComplete(mimeJavaStrings) : "Expected all of {" + String.join(", ", mimeJavaStrings) + "} in the PythonLanguage characterMimeTypes";
-    }
-
     public static final TruffleString[] T_DEFAULT_PYTHON_EXTENSIONS = new TruffleString[]{T_PY_EXTENSION, tsLiteral(".pyc")};
 
     public static final TruffleLogger LOGGER = TruffleLogger.getLogger(ID, PythonLanguage.class);
@@ -515,6 +467,11 @@ protected OptionDescriptors getOptionDescriptors() {
         return PythonOptions.DESCRIPTORS;
     }
 
+    @Override
+    protected OptionDescriptors getSourceOptionDescriptors() {
+        return PythonSourceOptions.DESCRIPTORS;
+    }
+
     @Override
     protected void initializeContext(PythonContext context) {
         if (!isLanguageInitialized) {
@@ -539,64 +496,56 @@ private synchronized void initializeLanguage() {
         }
     }
 
-    private static String optFlagsToMime(int optimize, int flags) {
-        if (optimize < 0) {
-            optimize = 0;
-        } else if (optimize > 2) {
-            optimize = 2;
-        }
-        String optField = new String(new byte[]{(byte) optimize});
-        String flagField = new String(new int[]{(flags & FutureFeature.ALL_FLAGS) >> MIME_FLAG_SHIFTBY}, 0, 1);
-        assert flagField.length() == 1 : "flags in mime type ended up a surrogate";
-        return optField + flagField;
-    }
-
-    public static String getCompileMimeType(int optimize, int flags) {
-        String optFlags = optFlagsToMime(optimize, flags);
-        return MIME_PREFIX + optFlags + "-compile";
-    }
-
-    public static String getEvalMimeType(int optimize, int flags) {
-        String optFlags = optFlagsToMime(optimize, flags);
-        return MIME_PREFIX + optFlags + "-eval";
+    public static SourceBuilder setPythonOptions(SourceBuilder sourceBuilder, InputType kind, int optimize, int flags) {
+        String sourceKind = switch (kind) {
+            case FILE -> "file";
+            case EVAL -> "eval";
+            case SINGLE -> "single";
+            default -> throw CompilerDirectives.shouldNotReachHere("unsupported source kind: " + kind);
+        };
+        return sourceBuilder.mimeType(PythonLanguage.MIME_TYPE) //
+                        .option("python.Optimize", Integer.toString(optimize)) //
+                        .option("python.Flags", Integer.toString(flags & FutureFeature.ALL_FLAGS)) //
+                        .option("python.Kind", sourceKind);
     }
 
     @Override
     protected CallTarget parse(ParsingRequest request) {
         PythonContext context = PythonContext.get(null);
         Source source = request.getSource();
-        if (source.getMimeType() == null || MIME_TYPE.equals(source.getMimeType())) {
-            if (!request.getArgumentNames().isEmpty() && source.isInteractive()) {
-                throw new IllegalStateException("parse with arguments not allowed for interactive sources");
-            }
-            InputType inputType = source.isInteractive() ? InputType.SINGLE : InputType.FILE;
-            return parse(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames(), EnumSet.noneOf(FutureFeature.class));
+        if (!request.getArgumentNames().isEmpty() && source.isInteractive()) {
+            throw new IllegalStateException("parse with arguments not allowed for interactive sources");
         }
-        if (!request.getArgumentNames().isEmpty()) {
-            throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
-        }
-
-        String mime = source.getMimeType();
-        String prefix = mime.substring(0, MIME_PREFIX.length());
-        if (!prefix.equals(MIME_PREFIX)) {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + mime);
-        }
-        String kind = mime.substring(MIME_KIND_START);
-        InputType type;
-        if (kind.equals(MIME_KIND_COMPILE)) {
-            type = InputType.FILE;
-        } else if (kind.equals(MIME_KIND_EVAL)) {
-            type = InputType.EVAL;
+        InputType inputType;
+        int optimize;
+        EnumSet<FutureFeature> futureFeatures;
+        boolean topLevel;
+        List<String> argumentNames;
+        boolean interactiveTerminal;
+        if (source.isInteractive()) {
+            inputType = InputType.SINGLE;
+            optimize = 0;
+            futureFeatures = EnumSet.noneOf(FutureFeature.class);
+            topLevel = true;
+            argumentNames = request.getArgumentNames();
+            interactiveTerminal = true;
         } else {
-            throw CompilerDirectives.shouldNotReachHere("unknown compilation kind: " + kind + " from mime type: " + mime);
-        }
-        int optimize = mime.codePointAt(MIME_PREFIX.length());
-        int flags = mime.codePointAt(MIME_PREFIX.length() + 1) << MIME_FLAG_SHIFTBY;
-        if (0 > optimize || optimize > 2 || (flags & ~FutureFeature.ALL_FLAGS) != 0) {
-            throw CompilerDirectives.shouldNotReachHere("Invalid value for optlevel or flags: " + optimize + "," + flags + " from mime type: " + mime);
+            var sourceOptions = source.getOptions(this);
+            String kind = sourceOptions.get(PythonSourceOptions.Kind);
+            topLevel = kind.isEmpty();
+            inputType = switch (kind) {
+                case "", "file" -> InputType.FILE;
+                case "eval" -> InputType.EVAL;
+                case "single" -> InputType.SINGLE;
+                default -> throw CompilerDirectives.shouldNotReachHere("unknown compilation kind: " + kind);
+            };
+            optimize = sourceOptions.get(PythonSourceOptions.Optimize);
+            int flags = sourceOptions.get(PythonSourceOptions.Flags);
+            futureFeatures = FutureFeature.fromFlags(flags);
+            argumentNames = request.getArgumentNames().isEmpty() ? null : request.getArgumentNames();
+            interactiveTerminal = false;
         }
-        assert !source.isInteractive();
-        return parse(context, source, type, false, optimize, false, null, FutureFeature.fromFlags(flags));
+        return parse(context, source, inputType, topLevel, optimize, interactiveTerminal, argumentNames, futureFeatures);
     }
 
     public static RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) {
@@ -952,7 +901,7 @@ public static TruffleLogger getCompatibilityLogger(Class<?> clazz) {
         return TruffleLogger.getLogger(ID, "compatibility." + clazz.getName());
     }
 
-    public static Source newSource(PythonContext ctxt, TruffleString tsrc, TruffleString name, boolean mayBeFile, String mime) {
+    public static Source newSource(PythonContext ctxt, TruffleString tsrc, TruffleString name, boolean mayBeFile, InputType inputType, int optimize, int flags) {
         try {
             SourceBuilder sourceBuilder = null;
             String src = tsrc.toJavaStringUncached();
@@ -977,9 +926,7 @@ public static Source newSource(PythonContext ctxt, TruffleString tsrc, TruffleSt
             if (sourceBuilder == null) {
                 sourceBuilder = Source.newBuilder(ID, src, name.toJavaStringUncached());
             }
-            if (mime != null) {
-                sourceBuilder.mimeType(mime);
-            }
+            sourceBuilder = PythonLanguage.setPythonOptions(sourceBuilder, inputType, optimize, flags);
             return newSource(ctxt, sourceBuilder);
         } catch (IOException e) {
             throw new IllegalStateException(e);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

@@ -1030,7 +1030,7 @@ Object compile(TruffleString expression, TruffleString filename, TruffleString m
                 }
             }
             if ((flags & PyCF_ONLY_AST) != 0) {
-                Source source = PythonLanguage.newSource(context, code, filename, mayBeFromFile, PythonLanguage.MIME_TYPE);
+                Source source = PythonLanguage.newSource(context, code, filename, mayBeFromFile, InputType.FILE, optimize, flags);
                 ParserCallbacksImpl parserCb = new ParserCallbacksImpl(source, PythonOptions.isPExceptionWithJavaStacktrace(getLanguage()));
 
                 EnumSet<AbstractParser.Flags> compilerFlags = EnumSet.noneOf(AbstractParser.Flags.class);
@@ -1054,14 +1054,10 @@ Object compile(TruffleString expression, TruffleString filename, TruffleString m
             CallTarget ct;
             TruffleString finalCode = code;
             Supplier<CallTarget> createCode = () -> {
-                if (type == InputType.FILE) {
-                    Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, PythonLanguage.getCompileMimeType(optimize, flags));
-                    return context.getEnv().parsePublic(source);
-                } else if (type == InputType.EVAL) {
-                    Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, PythonLanguage.getEvalMimeType(optimize, flags));
+                Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, type, optimize, flags);
+                if (type != InputType.SINGLE) {
                     return context.getEnv().parsePublic(source);
                 } else {
-                    Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, PythonLanguage.MIME_TYPE);
                     boolean allowIncomplete = (flags & PyCF_ALLOW_INCOMPLETE_INPUT) != 0;
                     return context.getLanguage().parse(context, source, InputType.SINGLE, false, optimize, false, allowIncomplete, null, FutureFeature.fromFlags(flags));
                 }
@@ -1211,7 +1207,7 @@ private TruffleString doDecodeSource(Object source, TruffleString filename, byte
         private RuntimeException raiseInvalidSyntax(TruffleString filename, String format, Object... args) {
             PythonContext context = getContext();
             // Create non-empty source to avoid overwriting the message with "unexpected EOF"
-            Source source = PythonLanguage.newSource(context, T_SPACE, filename, mayBeFromFile, null);
+            Source source = PythonLanguage.newSource(context, T_SPACE, filename, mayBeFromFile, InputType.FILE, 0, 0);
             SourceRange sourceRange = new SourceRange(1, 0, 1, 0);
             TruffleString message = toTruffleStringUncached(String.format(format, args));
             throw raiseSyntaxError(ParserCallbacks.ErrorType.Syntax, sourceRange, message, source, PythonOptions.isPExceptionWithJavaStacktrace(context.getLanguage()));