From 3f02c50b43a59d80b264cc699d7375d428061804 Mon Sep 17 00:00:00 2001 From: Sundararajan Athijegannathan Date: Thu, 21 May 2026 11:07:53 +0530 Subject: [PATCH] 13: Add a remote engine variant to evaluate python script in a remote process --- samples/RemoteEngine.java | 96 +++ samples/RemoteObject.java | 75 ++ samples/squaring.py | 34 + .../openjdk/engine/python/PythonConfig.java | 19 +- .../python/PythonRemoteCompiledScript.java | 135 ++++ .../python/PythonRemoteScriptEngine.java | 675 ++++++++++++++++++ .../engine/python/PythonScriptEngine.java | 26 +- .../engine/python/remote_script_engine.py | 349 +++++++++ .../engine/python/test/RemoteEngineTest.java | 359 ++++++++++ 9 files changed, 1766 insertions(+), 2 deletions(-) create mode 100644 samples/RemoteEngine.java create mode 100644 samples/RemoteObject.java create mode 100644 samples/squaring.py create mode 100644 src/main/java/org/openjdk/engine/python/PythonRemoteCompiledScript.java create mode 100644 src/main/java/org/openjdk/engine/python/PythonRemoteScriptEngine.java create mode 100644 src/main/resources/org/openjdk/engine/python/remote_script_engine.py create mode 100644 src/test/java/org/openjdk/engine/python/test/RemoteEngineTest.java diff --git a/samples/RemoteEngine.java b/samples/RemoteEngine.java new file mode 100644 index 0000000..843933d --- /dev/null +++ b/samples/RemoteEngine.java @@ -0,0 +1,96 @@ + +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * - Neither the name of Oracle nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import javax.script.*; + +import org.openjdk.engine.python.AbstractPythonScriptEngine.PyExecMode; +import org.openjdk.engine.python.PythonException; +import org.openjdk.engine.python.PythonRemoteCompiledScript; +import org.openjdk.engine.python.PythonRemoteScriptEngine; +import org.openjdk.engine.python.PythonScriptEngine; + + +void main() throws IOException { + System.setProperty("org.openjdk.engine.python.sys.prepend.path", ""); + var m = new ScriptEngineManager(); + var e = (PythonScriptEngine) m.getEngineByName("python"); + e.setExecMode(PyExecMode.FILE); + + try { + e.eval("import os"); + e.setExecMode(PyExecMode.EVAL); + IO.println("local pid = " + e.eval("os.getpid()")); + + e.setExecMode(PyExecMode.FILE); + e.eval("from squaring import *"); + IO.println(e.invokeFunction("square", 27)); + e.setExecMode(PyExecMode.EVAL); + var numbers = e.eval("[23, 44, 12]"); + + var re = PythonRemoteScriptEngine.create(e); + re.setExecMode(PyExecMode.FILE); + re.eval("import os"); + re.setExecMode(PyExecMode.EVAL); + IO.println("remote pid = " + re.eval("os.getpid()")); + re.setExecMode(PyExecMode.FILE); + re.eval("from squaring import *"); + // we can call global functions on the remote engine + IO.println(re.invokeFunction("square", 27)); + // we can call global functions on the remote engine by passing + // arbitaray Python picklable arguments from the local engine + IO.println("sum of " + numbers + " is " + re.invokeFunction("sum", numbers)); + + // we can compile script remotely and eval it many times + var reCompiled = (PythonRemoteCompiledScript) re.compile(""" + for i in range(10): + print(i*i) + """); + + IO.println(reCompiled); + reCompiled.eval(); + + // closed the compiled script + reCompiled.close(); + + IO.println(reCompiled); + // cannot eval after close! + reCompiled.eval(); + } catch (ScriptException se) { + if (se instanceof PythonException pe) { + pe.print(); + } else { + se.printStackTrace(); + } + } catch (NoSuchMethodException nsme) { + nsme.printStackTrace(); + } +} diff --git a/samples/RemoteObject.java b/samples/RemoteObject.java new file mode 100644 index 0000000..edfd560 --- /dev/null +++ b/samples/RemoteObject.java @@ -0,0 +1,75 @@ + +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * - Neither the name of Oracle nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import javax.script.*; + +import org.openjdk.engine.python.AbstractPythonScriptEngine.PyExecMode; +import org.openjdk.engine.python.PythonRemoteScriptEngine; +import org.openjdk.engine.python.PythonScriptEngine; + +// By default remote engine sends pickled objects. If you call +// any method on such objects, methods are executed locally. +// But, sometimes we may to access method(s) of a remote object +// so that methods will run on the remote process. +void main() throws Exception { + var m = new ScriptEngineManager(); + var e = (PythonScriptEngine) m.getEngineByName("python"); + e.setExecMode(PyExecMode.FILE); + + var re = PythonRemoteScriptEngine.create(e); + re.setExecMode(PyExecMode.FILE); + re.eval("import os"); + re.eval(""" + class MyClass: + def __init__(self, x): + self.x = x + def add(self, y): + print("pid =", os.getpid()) + print(self.x + y) + def call(self): + print("pid =", os.getpid()) + return self.x + """); + + re.setExecMode(PyExecMode.EVAL); + + // script has to register an object as remote object! + var remoteObj = re.eval("RemoteObjectManager.register(MyClass(25))"); + IO.println(remoteObj); + + // call "func" method on the remote object + re.invokeMethod(remoteObj, "add", 233); + + // get Java interface object backed by methods of remote object + var callable = re.getInterface(remoteObj, Callable.class); + IO.println(callable.call()); +} diff --git a/samples/squaring.py b/samples/squaring.py new file mode 100644 index 0000000..794fc3c --- /dev/null +++ b/samples/squaring.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# - Neither the name of Oracle nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +def square(x): + print(f"squaring {x} from process {os.getpid()}") + return x*x diff --git a/src/main/java/org/openjdk/engine/python/PythonConfig.java b/src/main/java/org/openjdk/engine/python/PythonConfig.java index 745fc05..a056fe0 100644 --- a/src/main/java/org/openjdk/engine/python/PythonConfig.java +++ b/src/main/java/org/openjdk/engine/python/PythonConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -126,11 +126,28 @@ private static OS getCurrentOS() { */ public static final boolean JAVASTACK_IN_PYEXCEPTION; + /** + * Perform Python GC before engine close. Default is true. + */ + public static final boolean PYTHON_GC_ON_CLOSE; + + /** + * Perform Java GC after engine close. Default is true. + */ + public static final boolean JAVA_GC_ON_CLOSE; + static { String propVal = System.getProperty("org.openjdk.engine.python.javastack_in_pyexception", "true"); JAVASTACK_IN_PYEXCEPTION = Boolean.parseBoolean(propVal); + + propVal = System.getProperty("org.openjdk.engine.python.gc.on.close", "true"); + PYTHON_GC_ON_CLOSE = Boolean.parseBoolean(propVal); + + propVal = System.getProperty("org.openjdk.engine.python.java.gc.on.close", "true"); + JAVA_GC_ON_CLOSE = Boolean.parseBoolean(propVal); } + /** * Returns the configured Python program name if explicitly set via the * "java.python.program.name" system property, otherwise null. diff --git a/src/main/java/org/openjdk/engine/python/PythonRemoteCompiledScript.java b/src/main/java/org/openjdk/engine/python/PythonRemoteCompiledScript.java new file mode 100644 index 0000000..8eb9a08 --- /dev/null +++ b/src/main/java/org/openjdk/engine/python/PythonRemoteCompiledScript.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.engine.python; + +import java.util.Objects; +import javax.script.CompiledScript; +import javax.script.ScriptContext; +import javax.script.ScriptException; + +/** + * A compiled Python code object evaluated by a remote Python interpreter and + * coordinated via a local {@link PythonRemoteScriptEngine}. + *

+ * The underlying code object lives in the remote interpreter; this wrapper + * allows evaluation and lifecycle control from Java. Calling {@link #close()} + * releases the remote resource and makes this instance unusable. + */ +public final class PythonRemoteCompiledScript extends CompiledScript implements AutoCloseable { + + private final PythonRemoteScriptEngine pyEngine; + private PyObject pyScript; + + /** + * Constructs a remote compiled script wrapper. + * + * @param pyEngine the remote script engine coordinating evaluation (non-null) + * @param scriptObj the remote CPython code object as a PyObject (non-null) + * @throws NullPointerException if either argument is null + */ + PythonRemoteCompiledScript(PythonRemoteScriptEngine pyEngine, PyObject scriptObj) { + this.pyEngine = Objects.requireNonNull(pyEngine); + this.pyScript = Objects.requireNonNull(scriptObj); + } + + /** + * Evaluates this compiled script using the provided script context in the + * remote interpreter. + * + * @param ctxt the script context providing bindings and I/O + * @return the result of evaluating the code object + * @throws ScriptException if remote evaluation fails + * @throws IllegalStateException if this compiled script has been closed + */ + @Override + public Object eval(ScriptContext ctxt) throws ScriptException { + checkClosed(); + return pyEngine.evalCompiled(pyScript, ctxt); + } + + /** + * Evaluates this compiled script using the engine's current context in the + * remote interpreter. + * + * @return the result of evaluating the code object + * @throws ScriptException if remote evaluation fails + * @throws IllegalStateException if this compiled script has been closed + */ + @Override + public synchronized Object eval() throws ScriptException { + checkClosed(); + return pyEngine.evalCompiled(pyScript); + } + + /** + * Returns the remote engine that produced this compiled script. + * + * @return the owning PythonRemoteScriptEngine + */ + @Override + public PythonRemoteScriptEngine getEngine() { + return pyEngine; + } + + /** + * Releases the remote code object and marks this instance as closed. Further + * evaluation attempts will throw IllegalStateException. This method is idempotent. + */ + @Override + public synchronized void close() { + if (pyScript != null) { + try { + pyEngine.closeCompiled(pyScript); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } finally { + pyScript.destroy(); + pyScript = null; + } + } + } + + /** + * Returns a debug-friendly string describing this compiled script. + * + * @return a string representation including the underlying remote code object + */ + @Override + public String toString() { + return String.format("PythonRemoteCompiledScript(%s)", pyScript); + } + + /** + * Ensures this compiled script has not been closed. + * + * @throws IllegalStateException if the script has been closed + */ + private void checkClosed() { + if (pyScript == null) { + throw new IllegalStateException("CompiledScript closed already"); + } + } +} diff --git a/src/main/java/org/openjdk/engine/python/PythonRemoteScriptEngine.java b/src/main/java/org/openjdk/engine/python/PythonRemoteScriptEngine.java new file mode 100644 index 0000000..7746b68 --- /dev/null +++ b/src/main/java/org/openjdk/engine/python/PythonRemoteScriptEngine.java @@ -0,0 +1,675 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.engine.python; + +import java.lang.reflect.Proxy; +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +import javax.script.Bindings; +import javax.script.CompiledScript; +import javax.script.ScriptContext; +import javax.script.ScriptEngineFactory; +import javax.script.ScriptException; + +/** + * ScriptEngine implementation that evaluates Python code in a remote Python interpreter + * while exposing a local Java API. A local {@link PythonScriptEngine} is used + * for conversions and as a bridge; the actual execution happens remotely via + * a remote client server protocol. + */ +public final class PythonRemoteScriptEngine extends AbstractPythonScriptEngine { + + static final String REMOTE_ENGINE_MODULE_NAME = "remote_script_engine"; + static final String REMOTE_ENGINE_MODULE_PATH; + + // Create a temporaray directory and file for the remote engine + // support module file remote_script_engine.py. + static { + try { + String moduleFileName = REMOTE_ENGINE_MODULE_NAME + ".py"; + + // create a temporary directory for new module + Path tempDir = Files.createTempDirectory("python-script-engine"); + + // create a new module .py file with the content from the .py resource bundled + try (var resIs = PythonScriptEngine.class.getResourceAsStream(moduleFileName)) { + Path tempFile = tempDir.resolve(moduleFileName); + tempFile.toFile().deleteOnExit(); + Files.write(tempFile, resIs.readAllBytes()); + } + tempDir.toFile().deleteOnExit(); + + // On Windows, Python infers \U as some unicode character and throws error + // for paths that contain \U (example: C:\User\tmp). So use / as file + // separator for all platforms. + REMOTE_ENGINE_MODULE_PATH = tempDir. + toAbsolutePath(). + toString(). + replace(File.separatorChar, '/'); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + private final PythonScriptEngine localPyEngine; + private final PyObject remoteClient; + + PythonRemoteScriptEngine(PythonScriptEngine localPyEnigne) throws ScriptException { + this.localPyEngine = Objects.requireNonNull(localPyEnigne); + PyObject remoteEngineConstr = initRemoteEngineClientConstructor().unregister(); + this.remoteClient = remoteEngineConstr.call().unregister(); + } + + // remote support + /** + * Creates a new remote sctipt engine backed by a remote Python interpreter, + * using the given engine as the local bridge for conversions and coordination. + *

+ * When the local ScriptEngine is closed, all remote script engines created using + * that ScriptEngine are also closed. + * + * @param pyEngine local PythonScriptEngine associated with the new remote engine. + * @return a new AbstractPythonScriptEngine that executes remotely + * @throws ScriptException if initialization of the remote client fails + */ + public static AbstractPythonScriptEngine create(PythonScriptEngine pyEngine) throws ScriptException { + // The remote client object is managed by PythonRemoteScriptEngine object. + final var remoteEngine = new PythonRemoteScriptEngine(pyEngine); + pyEngine.addDependentEngine(remoteEngine); + return remoteEngine; + } + + /** + * Evaluates the provided script in the remote interpreter using the supplied context. + * + * @param script the Python source code to execute (non-null) + * @param ctxt the ScriptContext providing globals and I/O + * @return the result of evaluation as a PyObject or converted Java value + * @throws ScriptException if remote evaluation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object eval(String script, ScriptContext ctxt) throws ScriptException { + checkClosed(); + CompilationState cs = newCompilationState(script, ctxt); + return remoteClient.callMethod("eval_command", + script, cs.name(), getCompileMode(cs.mode()), getPyDictionary(ctxt)); + } + + /** + * Evaluates the script read from the given Reader in the remote interpreter + * using the supplied context. + * + * @param reader Reader supplying Python source (will be fully read) + * @param ctxt the ScriptContext providing globals and I/O + * @return the result of evaluation + * @throws ScriptException if reading or remote evaluation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object eval(Reader reader, ScriptContext ctxt) throws ScriptException { + checkClosed(); + try { + return eval(readAll(reader), ctxt); + } catch (IOException ex) { + throw new ScriptException(ex); + } + } + + /** + * Evaluates the provided script in the remote interpreter using this engine's current context. + * + * @param script the Python source code to execute (non-null) + * @return the result of evaluation + * @throws ScriptException if remote evaluation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object eval(String script) throws ScriptException { + checkClosed(); + CompilationState cs = newCompilationState(script, context); + return remoteClient.callMethod("eval_command", + script, cs.name(), getCompileMode(cs.mode())); + } + + /** + * Evaluates the script read from the given Reader in the remote interpreter + * using this engine's current context. + * + * @param reader Reader supplying Python source (will be fully read) + * @return the result of evaluation + * @throws ScriptException if reading or remote evaluation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object eval(Reader reader) throws ScriptException { + checkClosed(); + try { + return eval(readAll(reader)); + } catch (IOException ioExp) { + throw new ScriptException(ioExp); + } + } + + /** + * Sets a global variable in the remote interpreter's globals dictionary. + * + * @param key variable name + * @param value Java value to store (converted to PyObject remotely) + * @throws RuntimeException wrapping ScriptException if the call fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized void put(String key, Object value) { + checkClosed(); + try { + remoteClient.callMethod("put_var_command", key, value); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Retrieves a global variable from the remote interpreter's globals dictionary. + * + * @param key variable name + * @return the value as a PyObject or converted Java value + * @throws RuntimeException wrapping ScriptException if the call fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object get(String key) { + checkClosed(); + try { + return remoteClient.callMethod("get_var_command", key); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Creates new ENGINE_SCOPE bindings. The returned bindings are local but + * compatible with the remote engine. + * + * @return new PythonBindings instance + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Bindings createBindings() { + checkClosed(); + return localPyEngine.createBindings(); + } + + /** + * Sets the ScriptContext and synchronizes the remote interpreter's globals. + * + * @param ctxt new ScriptContext whose ENGINE_SCOPE must be PythonBindings + * @throws RuntimeException wrapping ScriptException if synchronization fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized void setContext(ScriptContext ctxt) { + checkClosed(); + Objects.requireNonNull(ctxt); + // check that ENGINE_SCOPE is set acceptable value. + PythonBindings pyBindings = getPythonBindings(ctxt); + try { + remoteClient.callMethod("put_globals_command", pyBindings.getPyDictionary()); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } + super.setContext(ctxt); + } + + /** + * Returns the ScriptContext after synchronizing its ENGINE_SCOPE with the + * remote interpreter's globals. + * + * @return current ScriptContext with up-to-date globals + * @throws RuntimeException wrapping ScriptException if synchronization fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized ScriptContext getContext() { + checkClosed(); + try { + var pyDict = (PyDictionary) remoteClient.callMethod("get_globals_command"); + context.setBindings(new PythonBindings(pyDict), ScriptContext.ENGINE_SCOPE); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } + return context; + } + + /** + * Returns the ScriptEngineFactory associated with this engine (delegated to the local engine). + * + * @return factory instance + */ + @Override + public ScriptEngineFactory getFactory() { + return localPyEngine.getFactory(); + } + + /** + * Closes this engine and releases remote resources. Idempotent. + * + * @throws RuntimeException wrapping ScriptException if remote close fails + */ + @Override + public synchronized void close() { + if (this.closed) { + return; + } + try { + remoteClient.callMethod("engine_close_command"); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } finally { + remoteClient.destroy(); + if (PythonConfig.PYTHON_GC_ON_CLOSE) { + localPyEngine.gc(); + } + this.closed = true; + } + if (PythonConfig.JAVA_GC_ON_CLOSE) { + System.gc(); + } + } + + /** + * Remote engines are never main engines. + * + * @return false always + */ + @Override + public boolean isMainEngine() { + return false; + } + + /** + * Compiles the given script in the remote interpreter. + * + * @param script Python source (non-null) + * @return a compiled script wrapper executing remotely + * @throws ScriptException if compilation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized CompiledScript compile(String script) throws ScriptException { + checkClosed(); + CompilationState cs = newCompilationState(script, context); + try { + var pyScript = remoteClient.callMethod("compile_command", + script, cs.name(), getCompileMode(cs.mode())); + pyScript.unregister(); + return new PythonRemoteCompiledScript(this, pyScript); + } catch (ScriptException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Compiles the given Reader content in the remote interpreter. + * + * @param reader Reader supplying Python source + * @return a compiled script wrapper executing remotely + * @throws ScriptException if reading or compilation fails + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized CompiledScript compile(Reader reader) throws ScriptException { + checkClosed(); + try { + return compile(readAll(reader)); + } catch (IOException ioExp) { + throw new ScriptException(ioExp); + } + } + + /** + * Invokes a method on a Python object in the remote interpreter. + * + * @param thiz target object (converted if needed) + * @param name method name + * @param args arguments to pass + * @return the invocation result + * @throws ScriptException if the invocation fails remotely + * @throws NoSuchMethodException if the method cannot be resolved + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object invokeMethod(Object thiz, String name, Object... args) + throws ScriptException, NoSuchMethodException { + checkClosed(); + var pyAddr = localPyEngine.withPyObjectManager(() -> { + PyObject pyObj = remoteClient.callMethod("call_object_method", + fromJava(thiz), name, newPyList(toPyObjects(args))); + // make the result is not confined to the current PyObjectArena! + pyObj.unregister(); + // This transfers the object address! + return pyObj.addr(); + }); + return localPyEngine.wrap(pyAddr); + } + + /** + * Invokes a global function in the remote interpreter. + * + * @param name function name + * @param args arguments to pass + * @return the invocation result + * @throws ScriptException if the invocation fails remotely + * @throws NoSuchMethodException if the function cannot be resolved + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized Object invokeFunction(String name, Object... args) + throws ScriptException, NoSuchMethodException { + checkClosed(); + + var pyAddr = localPyEngine.withPyObjectManager(() -> { + PyObject pyObj = remoteClient.callMethod("call_global_function", + name, newPyList(toPyObjects(args))); + // make the result is not confined to the current PyObjectArena! + pyObj.unregister(); + // This transfers the object address! + return pyObj.addr(); + }); + return localPyEngine.wrap(pyAddr); + } + + /** + * Returns a Java proxy implementing the given interface by delegating to + * functions in the remote interpreter's globals. + * + * @param interface type + * @param iface public interface to implement + * @return a proxy instance + * @throws IllegalArgumentException if iface is not a public interface + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized T getInterface(Class iface) { + checkClosed(); + checkInterface(iface); + return implementInterface(iface); + } + + /** + * Returns a Java proxy implementing the given interface by delegating to + * methods on the specified Python object in the remote interpreter. + * + * @param interface type + * @param thiz Python object to dispatch calls to + * @param iface public interface to implement + * @return a proxy instance + * @throws IllegalArgumentException if iface is not a public interface + * @throws IllegalStateException if this engine is closed + */ + @Override + public synchronized T getInterface(Object thiz, Class iface) { + checkClosed(); + checkInterface(iface); + return implementInterface(thiz, iface); + } + + /** + * Converts a Java String to a remote Python str (delegated to local engine). + */ + @Override + public PyObject fromJava(String str) throws ScriptException { + return localPyEngine.fromJava(str); + } + + /** + * Converts a Java long to a remote Python int (delegated to local engine). + */ + @Override + public PyObject fromJava(long l) throws ScriptException { + return localPyEngine.fromJava(l); + } + + /** + * Converts a Java double to a remote Python float (delegated to local engine). + */ + @Override + public PyObject fromJava(double d) throws ScriptException { + return localPyEngine.fromJava(d); + } + + /** + * Converts a Java boolean to a remote Python bool (delegated to local engine). + */ + @Override + public PyObject fromJava(boolean b) throws ScriptException { + return localPyEngine.fromJava(b); + } + + /** + * Converts a supported Java object to a remote Python object (delegated to local engine). + */ + @Override + public PyObject fromJava(Object obj) throws ScriptException { + if (obj instanceof PyJavaFunction.Func pyFunc) { + return fromJava(pyFunc, pyFunc.toString(), null); + } else { + return localPyEngine.fromJava(obj); + } + } + + /** + * Converts a Java function as a Python function object. + */ + @Override + public PyJavaFunction fromJava(PyJavaFunction.Func func, String name, String doc) { + throw new UnsupportedOperationException("not implemented for remote engine"); + } + + /** + * Converts a Python-backed value to the requested Java type (delegated to local engine). + */ + @Override + public Object toJava(Object obj, Class cls) throws ScriptException { + return localPyEngine.toJava(obj, cls); + } + + /** + * Returns the Python None singleton (delegated to local engine). + */ + @Override + public PyConstant getNone() { + return localPyEngine.getNone(); + } + + /** + * Returns the Python False singleton (delegated to local engine). + */ + @Override + public PyConstant getFalse() { + return localPyEngine.getFalse(); + } + + /** + * Returns the Python True singleton (delegated to local engine). + */ + @Override + public PyConstant getTrue() { + return localPyEngine.getTrue(); + } + + /** + * Returns the Python Ellipsis singleton (delegated to local engine). + */ + @Override + public PyConstant getEllipsis() { + return localPyEngine.getEllipsis(); + } + + /** + * Returns the Python NotImplemented singleton (delegated to local engine). + */ + @Override + public PyConstant getNotImplemented() { + return localPyEngine.getNotImplemented(); + } + + /** + * Creates a new Python dict (delegated to local engine). + */ + @Override + public PyDictionary newPyDictionary() throws ScriptException { + return localPyEngine.newPyDictionary(); + } + + /** + * Creates a new Python list with the provided items (delegated to local engine). + */ + @Override + public PyList newPyList(PyObject... items) throws ScriptException { + return localPyEngine.newPyList(items); + } + + /** + * Creates a new Python tuple with the provided items (delegated to local engine). + */ + @Override + public PyTuple newPyTuple(PyObject... items) throws ScriptException { + return localPyEngine.newPyTuple(items); + } + + // package private helpers below this point + Object evalCompiled(PyObject pyScript, ScriptContext ctxt) throws ScriptException { + return remoteClient.callMethod("eval_codeobject_command", pyScript, getPyDictionary(ctxt)); + } + + Object evalCompiled(PyObject pyScript) throws ScriptException { + return remoteClient.callMethod("eval_codeobject_command", pyScript); + } + + Object closeCompiled(PyObject pyScript) throws ScriptException { + return remoteClient.callMethod("close_codeobject_command", pyScript); + } + + // internals only below this point + private PyObject initRemoteEngineClientConstructor() throws ScriptException { + // create new ScriptContext to avoid polluting the global scope. + try (var pyBindings = (PythonBindings) localPyEngine.createBindings()) { + var newContext = localPyEngine.getScriptContext(pyBindings); + PyExecMode oldMode = execMode; + try { + localPyEngine.setExecMode(PyExecMode.FILE); + + // append the temporary directory to module search path + localPyEngine.eval(String.format(""" + import sys + sys.path.append("%s") + """, PythonRemoteScriptEngine.REMOTE_ENGINE_MODULE_PATH), newContext); + + // import RemoteEngineClient class + localPyEngine.eval(String.format("from %s import RemoteEngineClient", + PythonRemoteScriptEngine.REMOTE_ENGINE_MODULE_NAME), newContext); + + localPyEngine.setExecMode(PyExecMode.EVAL); + // return the RemoteEngineClass class to use as constructor later + return (PyObject) localPyEngine.eval("RemoteEngineClient", newContext); + } finally { + localPyEngine.setExecMode(oldMode); + } + } + } + + private T implementInterface(Class iface) { + return iface.cast(Proxy.newProxyInstance(iface.getClassLoader(), + new Class[]{iface}, + (proxy, method, args) -> { + if (args == null) { + args = new Object[0]; + } + + var value = invokeFunction(method.getName(), args); + return toJava(value, method.getReturnType()); + } + )); + } + + private T implementInterface(Object thiz, Class iface) { + return iface.cast(Proxy.newProxyInstance(iface.getClassLoader(), + new Class[]{iface}, + (proxy, method, args) -> { + if (args == null) { + args = new Object[0]; + } + + var value = invokeMethod(thiz, method.getName(), args); + return toJava(value, method.getReturnType()); + } + )); + } + + /** + * Converts arbitrary Java arguments into PyObject values using the local engine, + * suitable for transmission/usage on the remote interpreter. + * + * Ownership semantics follow the local engine rules; callers must ensure appropriate + * lifetimes for any temporary values on the remote side. + * + * @param args Java arguments to convert + * @return array of PyObject values suitable for remote calls + * @throws ScriptException if any conversion fails + */ + private PyObject[] toPyObjects(Object... args) throws ScriptException { + PyObject[] res = new PyObject[args.length]; + for (int i = 0; i < args.length; i++) { + res[i] = fromJava(args[i]); + } + return res; + } + + // code mode string as passed to Python builtin function "compile" + /** + * Maps the engine's compile mode to the string accepted by Python's builtin + * compile() function on the remote interpreter. + * + * @param mode engine execution/compile mode + * @return "single" for SINGLE, "exec" for FILE, "eval" for EVAL + */ + private String getCompileMode(PyExecMode mode) { + return switch (mode) { + case SINGLE -> + "single"; + case FILE -> + "exec"; + case EVAL -> + "eval"; + }; + } +} diff --git a/src/main/java/org/openjdk/engine/python/PythonScriptEngine.java b/src/main/java/org/openjdk/engine/python/PythonScriptEngine.java index 57ede73..b0bc18a 100644 --- a/src/main/java/org/openjdk/engine/python/PythonScriptEngine.java +++ b/src/main/java/org/openjdk/engine/python/PythonScriptEngine.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, 2026 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -672,6 +672,24 @@ public PyTuple newPyTuple(PyObject... items) throws ScriptException { return new PyTuple(this, items); } + /** + * Perform Python GC. + */ + public synchronized void gc() { + var oldMode = getExecMode(); + setExecMode(PyExecMode.FILE); + try { + eval("import gc; gc.collect()"); + } catch (ScriptException se) { + if (PythonConfig.DEBUG) { + IO.println("python gc failed!"); + se.printStackTrace(System.out); + } + } finally { + setExecMode(oldMode); + } + } + // AutoClosable @Override public synchronized void close() { @@ -680,6 +698,9 @@ public synchronized void close() { return; } closeAllDependentEngines(); + if (PythonConfig.PYTHON_GC_ON_CLOSE) { + gc(); + } EngineLifeCycleManager.close(this); this.closed = true; this.pyInterpreterState = null; @@ -694,6 +715,9 @@ public synchronized void close() { this.pyMethodDefMap.clear(); this.engineArena = null; this.dependentEngines.clear(); + if (PythonConfig.JAVA_GC_ON_CLOSE) { + System.gc(); + } } // package-private helpers below this point diff --git a/src/main/resources/org/openjdk/engine/python/remote_script_engine.py b/src/main/resources/org/openjdk/engine/python/remote_script_engine.py new file mode 100644 index 0000000..cbfe80d --- /dev/null +++ b/src/main/resources/org/openjdk/engine/python/remote_script_engine.py @@ -0,0 +1,349 @@ +## +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Oracle designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +## + + +import builtins +import multiprocessing +import pickle +import sys +import typing + +from enum import Enum +from multiprocessing import Queue +from typing import NamedTuple + +multiprocessing.set_start_method("spawn", force=True) + +def is_picklable(value): + """Checks if a value is picklable.""" + try: + pickle.dumps(value) + return True + except (pickle.PicklingError, TypeError, AttributeError): + # Catches general pickling errors and TypeErrors (e.g., for lambda functions) + return False + +def picklable_dict(original_dict: dict): + if is_picklable(original_dict): + return original_dict + else: + return { + key: value + for key, value in original_dict.items() + if key != "__builtins__" and is_picklable(value) + } + +def picklable_list(original_list: list): + if is_picklable(original_list): + return original_list + else: + return [ picklable_value(value) for value in original_list ] + +def picklable_tuple(original_tuple: tuple): + if is_picklable(original_tuple): + return original_tuple + else: + temp_list = [ picklable_value(value) for value in original_tuple ] + # handle tuple subclasses like NamedTuple + return type(original_tuple)(temp_list) + +def picklable_value(value): + if isinstance(value, dict): + return picklable_dict(value) + elif isinstance(value, list): + return picklable_list(value) + elif isinstance(value, tuple): + return picklable_tuple(value) + else: + return value if is_picklable(value) else None + +# command type codes +class CommandType(Enum): + DONE = 0 + # command data tuple (file_name, mode, scope_dict) + EVAL = 1 + # command data tuple (file_name, mode) + COMPILE = 2 + # (optional) command data is scope dictionary + EVAL_COMPILED = 3 + CLOSE_COMPILED = 4 + GET_VAR = 5 + # command data is value of the variable assigned + PUT_VAR = 6 + GET_GLOBALS = 7 + # command data is scope dictionary + PUT_GLOBALS = 8 + # command data is the list of args + CALL_FUNCTION = 9 + # command data is the list of args including self + CALL_METHOD = 10 + +# remote command details +class Command(NamedTuple): + # type of the command + type: CommandType + # message string associated with the command + message: str + # command specific additional data, if any. can be None + data: object | None + +# result from a remote command +class Result(NamedTuple): + # result value + value: object | None + # exception value + error: Exception | None + +# result from eval commands that accept input dictionary +class EvalCommandResult(NamedTuple): + # eval result + eval_value: object | None + # output dictionary + scope_dict: dict | None + +# This is the token sent to client for a remote object +class RemoteObject(NamedTuple): + object_id: str + +# simple dictionary based remote object manager +class RemoteObjectManager: + # RemoteObject -> (local) object + objectMap = dict() + + @classmethod + def find(cls, remoteObj: RemoteObject) -> object: + return cls.objectMap.get(remoteObj, None) + + @classmethod + def register(cls, obj: object) -> RemoteObject: + remoteObj = RemoteObject(hex(id(obj))); + cls.objectMap[remoteObj] = obj; + return remoteObj + + @classmethod + def unregister(cls, remoteObj: RemoteObject) -> None: + cls.objectMap.pop(remoteObj, None) + +class RemoteEngineServer: + def __init__(self, command_queue: Queue, result_queue: Queue): + self.command_queue = command_queue + self.result_queue = result_queue + self.set_current_globals(dict()) + # string id -> code_object + self.code_objects = dict() + + def set_current_globals(self, d: dict): + d["__builtins__"] = builtins.__dict__ + # expose remote object manager for convenience + d["RemoteObjectManager"] = RemoteObjectManager + self.current_globals = d + + def get_global_function(self, func_name: str): + if func_name in self.current_globals: + return self.current_globals[func_name] + elif hasattr(builtins, func_name): + return getattr(builtins, func_name) + else: + return None + + def execute_command(self, command: Command): + match command.type: + case CommandType.EVAL: + # command.data tuple (file_name, mode, scope_dict) + (file_name, mode, scope_dict) = typing.cast(tuple, command.data) + if not isinstance(command.data, tuple): + raise TypeError("command.data is not a tuple") + code_object = compile(command.message, file_name, mode); + if isinstance(scope_dict, dict): + eval_value = eval(code_object, scope_dict, None) + # return a named tuple with eval value and the updated dictionary + return EvalCommandResult(picklable_value(eval_value), picklable_dict(scope_dict)) + else: + return picklable_value(eval(code_object, self.current_globals, None)) + case CommandType.COMPILE: + # command.data tuple (file_name, mode) + (file_name, mode) = typing.cast(tuple, command.data) + if not isinstance(command.data, tuple): + raise TypeError("command.data is not a tuple") + code_object = compile(command.message, file_name, mode); + code_object_id = hex(id(code_object)) + self.code_objects[code_object_id] = code_object + return code_object_id + case CommandType.EVAL_COMPILED: + # optional command data is scope dictionary + code_object = self.code_objects[command.message] + if isinstance(command.data, dict): + scope_dict = command.data + eval_value = eval(code_object, scope_dict, None) + # return a named tuple with eval value and the updated dictionary + return EvalCommandResult(picklable_value(eval_value), picklable_dict(scope_dict)) + else: + return picklable_value(eval(code_object, self.current_globals, None)) + case CommandType.CLOSE_COMPILED: + self.code_objects.pop(command.message, None) + return command.message + case CommandType.GET_VAR: + return picklable_value(self.current_globals[command.message]) + case CommandType.PUT_VAR: + # command data is value of the variable assigned + # do not allow __builtins__ to be overwritten! + if command.message != "__builtins__": + self.current_globals[command.message] = command.data + else: + raise Exception("cannot overwritte __builtins__") + return None + case CommandType.GET_GLOBALS: + return picklable_dict(self.current_globals) + case CommandType.PUT_GLOBALS: + # command data is scope dictionary + if isinstance(command.data, dict): + self.set_current_globals(command.data) + return None + else: + raise TypeError("command.data is not a dictionary") + case CommandType.CALL_FUNCTION: + # command data is the list of args + args = typing.cast(list, command.data) + if isinstance(args, list): + func = self.get_global_function(command.message) + if not callable(func): + raise TypeError("not a callable: " + command.message) + return picklable_value(func(*args)) + else: + raise TypeError("command.data is not a list") + case CommandType.CALL_METHOD: + # command data is the list of args including self + args = typing.cast(list, command.data) + remoteObj = args[0] + if not isinstance(remoteObj, RemoteObject): + raise TypeError("expected a RemoteObject in command.data[0]") + if isinstance(args, list): + localObj = RemoteObjectManager.find(remoteObj) + if localObj is None: + raise Exception("No such remote object: " + str(remoteObj)) + method = getattr(localObj, command.message) + if not callable(method): + raise TypeError("not a callable: " + command.message) + return picklable_value(method(*args[1:])) + else: + raise TypeError("command.data is not a list") + case _: + print(f"Unknown command: {command}") + return None + + def run(self): + exit_code = 0 + while True: + command = self.command_queue.get() + if command.type == CommandType.DONE: + self.result_queue.put(Result("DONE", None)) + if isinstance(command.data, int): + exit_code = command.data + break + try: + result = Result(self.execute_command(command), None); + except Exception as e: + result = Result(None, e) + self.result_queue.put(result) + self.command_queue.close() + self.result_queue.close() + sys.exit(exit_code) + + + +def server(command_queue: Queue, result_queue: Queue): + remote_engine_server = RemoteEngineServer(command_queue, result_queue) + remote_engine_server.run() + +class RemoteEngineClient: + def __init__(self): + self.command_queue = multiprocessing.Queue() + self.result_queue = multiprocessing.Queue() + + server_proc = multiprocessing.Process(target=server, args=(self.command_queue, self.result_queue)) + server_proc.start() + + def send_command(self, cmd: Command): + if not is_picklable(cmd.data): + raise Exception("cannot pickle command data") + self.command_queue.put(cmd) + result = self.result_queue.get() + if result.error is None: + return result.value + else: + raise result.error + + def eval_command(self, code: str, file_name: str, mode: str, scope_dict: dict | None = None): + if isinstance(scope_dict, dict): + eval_result = self.send_command(Command(CommandType.EVAL, code, (file_name, mode, picklable_dict(scope_dict)))) + scope_dict.update(eval_result.scope_dict) + return eval_result.eval_value; + else: + return self.send_command(Command(CommandType.EVAL, code, (file_name, mode, None))) + + def compile_command(self, code: str, file_name: str, mode: str = 'eval'): + return self.send_command(Command(CommandType.COMPILE, code, (file_name, mode))) + + def eval_codeobject_command(self, code: str, scope_dict: dict | None = None): + if isinstance(scope_dict, dict): + eval_result = self.send_command(Command(CommandType.EVAL_COMPILED, code, picklable_dict(scope_dict))) + scope_dict.update(eval_result.scope_dict) + return eval_result.eval_value + else: + return self.send_command(Command(CommandType.EVAL_COMPILED, code, None)) + + def close_codeobject_command(self, code: str): + return self.send_command(Command(CommandType.CLOSE_COMPILED, code, None)) + + def get_var_command(self, name: str): + return self.send_command(Command(CommandType.GET_VAR, name, None)) + + def put_var_command(self, name: str, value: object): + return self.send_command(Command(CommandType.PUT_VAR, name, picklable_value(value))) + + def get_globals_command(self): + return self.send_command(Command(CommandType.GET_GLOBALS, "", None)) + + def put_globals_command(self, scope_dict: dict): + if not isinstance(scope_dict, dict): + raise TypeError("expected a dict object for scope_dict") + return self.send_command(Command(CommandType.PUT_GLOBALS, "", picklable_dict(scope_dict))) + + def call_global_function(self, func_name: str, args: list): + if not isinstance(args, list): + raise TypeError("expected a list for args") + return self.send_command(Command(CommandType.CALL_FUNCTION, func_name, picklable_list(args))) + + def call_object_method(self, obj: RemoteObject, method_name: str, args: list): + if not isinstance(obj, RemoteObject): + raise TypeError("expected a RemoteObject") + if not isinstance(args, list): + raise TypeError("expected a list for args") + return self.send_command(Command(CommandType.CALL_METHOD, method_name, picklable_list([ obj, *args ]))) + + def engine_close_command(self): + self.send_command(Command(CommandType.DONE, "", None)) + self.command_queue.close() + self.result_queue.close() diff --git a/src/test/java/org/openjdk/engine/python/test/RemoteEngineTest.java b/src/test/java/org/openjdk/engine/python/test/RemoteEngineTest.java new file mode 100644 index 0000000..fd37281 --- /dev/null +++ b/src/test/java/org/openjdk/engine/python/test/RemoteEngineTest.java @@ -0,0 +1,359 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.engine.python.test; + +import java.util.concurrent.Callable; +import javax.script.*; +import org.openjdk.engine.python.*; +import org.openjdk.engine.python.AbstractPythonScriptEngine.PyExecMode; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class RemoteEngineTest { + + private PythonScriptEngine engine; + + @BeforeClass + public void createEngine() { + ScriptEngineManager m = new ScriptEngineManager(); + this.engine = (PythonScriptEngine) m.getEngineByName("python"); + } + + @AfterClass + public void closeEngine() { + this.engine.close(); + } + + @Test + public void testPidsNotEqual() throws ScriptException { + engine.setExecMode(PyExecMode.FILE); + engine.eval("import os"); + engine.setExecMode(PyExecMode.EVAL); + long localPid = ((PyObject)engine.eval("os.getpid()")).toLong(); + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval("import os"); + re.setExecMode(PyExecMode.EVAL); + long remotePid = ((PyObject)re.eval("os.getpid()")).toLong(); + assertTrue(localPid != remotePid); + } + } + + @Test + public void testEvalWithContext() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + var bindings = re.createBindings(); + var ctxt = new SimpleScriptContext(); + ctxt.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + + int x = 1028; + bindings.put("x", x); + int y = 3444; + bindings.put("y", y); + re.setExecMode(PyExecMode.FILE); + re.eval("z = x*y", ctxt); + re.setExecMode(PyExecMode.EVAL); + var evalValue = (PyObject) re.eval("z", ctxt); + assertEquals(evalValue.toLong(), x * y); + + // make sure context is updated properly + assertTrue(bindings.containsKey("x")); + assertTrue(bindings.containsKey("y")); + assertTrue(bindings.containsKey("z")); + + // We evaluated everything in a separate ScriptContext. + // The default context should be clean + var defBindings = re.getContext().getBindings(ScriptContext.ENGINE_SCOPE); + assertFalse(defBindings.containsKey("x")); + assertFalse(defBindings.containsKey("y")); + assertFalse(defBindings.containsKey("z")); + } + } + + @Test + public void testEvalWithContextNotPicklable() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + var bindings = re.createBindings(); + var ctxt = new SimpleScriptContext(); + ctxt.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + + var func = engine.fromJava((PyJavaFunction.NoArgFunc)() -> engine.getNone()); + int x = 1028; + bindings.put("x", x); + int y = 3444; + bindings.put("y", y); + // 'func' is not pickable and so 'w' is not set in + // to remote bindings + bindings.put("w", func); + re.setExecMode(PyExecMode.FILE); + re.eval("z = x*y", ctxt); + re.setExecMode(PyExecMode.EVAL); + var evalValue = (PyObject) re.eval("z", ctxt); + assertEquals(evalValue.toLong(), x * y); + + // make sure context is updated properly + assertTrue(bindings.containsKey("x")); + assertTrue(bindings.containsKey("y")); + assertTrue(bindings.containsKey("z")); + assertTrue(bindings.containsKey("w")); + + // We evaluated everything in a separate ScriptContext. + // The default context should be clean + var defBindings = re.getContext().getBindings(ScriptContext.ENGINE_SCOPE); + assertFalse(defBindings.containsKey("x")); + assertFalse(defBindings.containsKey("y")); + assertFalse(defBindings.containsKey("z")); + assertFalse(defBindings.containsKey("w")); + } + } + + @Test + public void testRemoteFunctionCall() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval(""" + def square(x): + return x*x + """); + re.setExecMode(PyExecMode.EVAL); + long squareValue = ((PyObject)re.eval("square(27)")).toLong(); + assertTrue(squareValue == 27*27L); + } + } + + @Test + public void testRemoteMethodCall() throws ScriptException, NoSuchMethodException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval(""" + class Adder: + def __init__(self, x): + self.x = x + def add(self, y): + if y is None: + return self.x + else: + return self.x + y + """); + + var func = engine.fromJava((PyJavaFunction.NoArgFunc)() -> engine.getTrue()); + re.setExecMode(PyExecMode.EVAL); + var remoteObj = re.eval("RemoteObjectManager.register(Adder(25))"); + var value = (PyObject) re.invokeMethod(remoteObj, "add", 233); + assertEquals(value.toLong(), 233 + 25); + // 'func' is not pickable and so None is sent. Adder takes care + // of None by returning self.x + value = (PyObject) re.invokeMethod(remoteObj, "add", func); + assertEquals(value.toLong(), 25); + } + } + + @Test + public void testRemoteMethodCallOnNotRemoteObject() throws ScriptException, NoSuchMethodException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.EVAL); + var remoteObj = re.eval("[334, 344]"); + // The following should throw exception, as List is not registered + // as a remote object and so received here as local after pickling. + boolean sawPyException = false; + try { + re.invokeMethod(remoteObj, "append", 33); + } catch (PythonException pe) { + assertTrue(pe.toString().contains("expected a RemoteObject")); + sawPyException = true; + } + assertTrue(sawPyException); + + re.setExecMode(PyExecMode.FILE); + re.eval("aList = [334, 344]"); + re.setExecMode(PyExecMode.EVAL); + // now do a proper remote registration + remoteObj = re.eval("RemoteObjectManager.register(aList)"); + re.invokeMethod(remoteObj, "append", 33); + var len = (PyObject) re.invokeMethod(remoteObj, "__len__"); + assertEquals(len.toLong(), 3); + } + } + + @Test + public void testGlobalVariableRead() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval("x = 'hello'"); + re.setExecMode(PyExecMode.EVAL); + var str = ((PyObject)re.get("x")).toString(); + assertEquals(str, "hello"); + } + } + + @Test + public void testGlobalVariableWrite() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + var func = engine.fromJava((PyJavaFunction.NoArgFunc)() -> engine.getNone()); + re.setExecMode(PyExecMode.FILE); + re.put("x", 23); + // 'func' is not picklable and so None is set in the remote + re.put("func", func); + re.setExecMode(PyExecMode.EVAL); + var value = ((PyObject)re.eval("x*x*x")).toLong(); + assertEquals(value, 23*23*23); + var obj = (PyObject)re.eval("func"); + assertTrue(obj.isNone()); + } + } + + @Test + public void testGlobalVariableWriteBuiltins() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + try { + re.put("__builtins__", 23); + throw new AssertionError("should not reach here"); + } catch (RuntimeException rx) { + assertTrue(rx.getMessage().contains("cannot overwritte __builtins__")); + } + } + } + + @Test + public void testGetGlobals() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval("x = 'hello'"); + re.eval("y = 23"); + Bindings bindings = re.getContext().getBindings(ScriptContext.ENGINE_SCOPE); + assertEquals(((PyObject)bindings.get("x")).toString(), "hello"); + assertEquals(((PyObject)bindings.get("y")).toLong(), 23L); + } + } + + @Test + public void testSetGlobals() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + var bindings = re.createBindings(); + ScriptContext sc = new SimpleScriptContext(); + int x = 34, y = 25, z = 23; + bindings.put("x", x); + bindings.put("y", y); + bindings.put("z", z); + sc.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + re.setContext(sc); + re.setExecMode(PyExecMode.EVAL); + long value = ((PyObject) re.eval("x + y + z")).toLong(); + assertEquals(value, x + y + z); + } + } + + @Test + public void testCompile() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.EVAL); + var compiledScript = (PythonRemoteCompiledScript) re.compile("89*89"); + var evalValue = (PyObject) compiledScript.eval(); + assertEquals(evalValue.toLong(), 89*89); + compiledScript.close(); + + re.setExecMode(PyExecMode.SINGLE); + compiledScript = (PythonRemoteCompiledScript) re.compile("import os; os.name"); + compiledScript.eval(); + + re.setExecMode(PyExecMode.FILE); + re.eval(""" + def save_sys_path(path): + global sys_path + sys_path = path + """); + + compiledScript.close(); + compiledScript = (PythonRemoteCompiledScript) re.compile("import sys; save_sys_path(sys.path)"); + compiledScript.eval(); + re.setExecMode(PyExecMode.EVAL); + assertTrue(re.eval("sys_path") instanceof PyList); + compiledScript.close(); + } + } + + @Test + public void testCompileWithContext() throws ScriptException { + try (var re = PythonRemoteScriptEngine.create(engine)) { + var bindings = re.createBindings(); + var ctxt = new SimpleScriptContext(); + ctxt.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + + int x = -343; + bindings.put("x", x); + int y = 423; + bindings.put("y", y); + var func = engine.fromJava((PyJavaFunction.NoArgFunc)() -> engine.getNone()); + // 'func' is not picklable - but that should be ignored! + bindings.put("func", func); + re.setExecMode(PyExecMode.EVAL); + var compiledScript = re.compile("x*y"); + var evalValue = (PyObject) compiledScript.eval(ctxt); + assertEquals(evalValue.toLong(), x*y); + } + } + + @Test + public void testGetInterface() throws Exception { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval(""" + def call(): + return "hello" + """); + + Callable callable = re.getInterface(Callable.class); + assertEquals(callable.call().toString(), "hello"); + } + } + + @Test + public void testGetInterfaceOnObject() throws Exception { + try (var re = PythonRemoteScriptEngine.create(engine)) { + re.setExecMode(PyExecMode.FILE); + re.eval(""" + class Callback: + def __init__(self, x): + self.x = x + def call(self): + return self.x + """); + + re.setExecMode(PyExecMode.EVAL); + var remoteObj = re.eval("RemoteObjectManager.register(Callback(25))"); + Callable callable = re.getInterface(remoteObj, Callable.class); + assertEquals(callable.call().toString(), "25"); + + remoteObj = re.eval("RemoteObjectManager.register(Callback('hello'))"); + callable = re.getInterface(remoteObj, Callable.class); + assertEquals(callable.call().toString(), "hello"); + } + } +}