diff --git a/.gitignore b/.gitignore index fb07e725f0..33f2a45200 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ build/ /data /resource +/python_services /bin /dist /repo diff --git a/pom.xml b/pom.xml index a1507381a2..6ce19fffbd 100644 --- a/pom.xml +++ b/pom.xml @@ -1,102 +1,102 @@ - - - 4.0.0 - org.myrobotlab - mrl - 0.0.1-SNAPSHOT - MyRobotLab - Open Source Creative Machine Control - - - false - - - - 1.1. - - ${maven.build.timestamp} - yyyyMMddHHmm - ${timestamp} - ${version.prefix}${build.number} - ${git.branch} - ${NODE_NAME} - ${NODE_LABELS} - - - - 11 - 11 - UTF-8 - - - + + + 4.0.0 + org.myrobotlab + mrl + 0.0.1-SNAPSHOT + MyRobotLab + Open Source Creative Machine Control + + + false + + + + 1.1. + + ${maven.build.timestamp} + yyyyMMddHHmm + ${timestamp} + ${version.prefix}${build.number} + ${git.branch} + ${NODE_NAME} + ${NODE_LABELS} + + + + 11 + 11 + UTF-8 + + + @@ -135,9 +135,9 @@ https://m2.dv8tion.net/releases - - - + + + javazoom @@ -200,12 +200,7 @@ - - org.bytedeco - javacpp - 1.5.7 - provided - + org.deeplearning4j deeplearning4j-core @@ -659,6 +654,30 @@ log4j log4j + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -675,6 +694,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -691,6 +734,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -707,6 +774,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -723,6 +814,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -747,6 +862,30 @@ log4j log4j + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -763,6 +902,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -779,6 +942,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -795,6 +982,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -811,6 +1022,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -827,6 +1062,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -843,6 +1102,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -859,6 +1142,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -875,6 +1182,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -891,6 +1222,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -907,6 +1262,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -923,10 +1302,6 @@ org.apache.httpcomponents httpclient - - org.slf4j - slf4j-api - commons-io commons-io @@ -951,10 +1326,6 @@ org.apache.opennlp opennlp-maxent - - org.slf4j - slf4j-log4j12 - @@ -1268,18 +1639,9 @@ 0.10.9.7 provided - - org.bytedeco - cpython-platform - 3.10.8-1.5.8 - provided - - - org.bytedeco - cpython - 3.10.8-1.5.8 - provided - + + + @@ -1386,6 +1748,26 @@ 3.9.0 + + org.bytedeco + cpython-platform + 3.10.8-1.5.8 + + + org.bytedeco + cpython + 3.10.8-1.5.8 + + + org.bytedeco + javacpp + 1.5.8 + + + org.bytedeco + javacpp-platform + 1.5.8 + @@ -1734,375 +2116,382 @@ - - - org.mockito - mockito-core - 3.12.4 - test - - - - - - - false - src/main/resources - - - false - src/main/java - - ** - - - **/*.java - - - - - - false - src/test/resources - - - false - src/test/java - - ** - - - **/*.java - - - - src/main/resources - ${project.basedir} - - - - - - - - org.codehaus.mojo - properties-maven-plugin - 1.0.0 - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.1.0 - - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - no-duplicate-declared-dependencies - - enforce - - - - - - - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - read-project-properties - - - - build.properties - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.1.0 - - - package - - shade - - - myrobotlab - - true - myrobotlab-full - false - - - - - org.myrobotlab.service.Runtime - ${version} - ${version} - - ${build.number} - ${maven.build.timestamp} - ${agent.name} - ${user.name} - - - ${git.tags} - ${git.branch} - ${git.dirty} - ${git.remote.origin.url} - ${git.commit.id} - ${git.commit.id.abbrev} - ${git.commit.id.full} - ${git.commit.id.describe} - ${git.commit.id.describe-short} - ${git.commit.user.name} - ${git.commit.user.email} - - ${git.commit.time} - ${git.closest.tag.name} - ${git.closest.tag.commit.count} - ${git.build.user.name} - ${git.build.user.email} - ${git.build.time} - ${git.build.version} - - - - - - - *:* - - module-info.class - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - assembly.xml - - myrobotlab - false - - - - trigger-assembly - package - - single - - - - - - - true - org.apache.maven.plugins - maven-compiler-plugin - 2.3.2 - - 11 - 11 - true - true - -parameters - - - - - org.apache.maven.plugins - maven-resources-plugin - 2.4.3 - - - - pl.project13.maven - git-commit-id-plugin - 4.9.10 - - - initialize - get-the-git-infos - - revision - - - - - ${project.basedir}/.git - git - false - true - ${project.build.outputDirectory}/git.properties - - - false - false - -dirty - - - - - - maven-surefire-plugin - org.apache.maven.plugins - 2.22.2 - - -Djava.library.path=libraries/native -Djna.library.path=libraries/native - - **/*Test.java - - - **/integration/* - - - - - - - - org.apache.maven.plugins - maven-clean-plugin - 2.3 - - - - data/.myrobotlab - false - - - libraries - - ** - - false - - - data - - ** - - - - resource - - ** - - - - src/main/resources/resource/framework - - **/serviceData.json - - false - - - - - - - - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - 2.22.2 - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.0.1 - - - - - myrobotlab - http://myrobotlab.org - - - github - https://github.com/MyRobotLab/myrobotlab/issues - - + + + org.mockito + mockito-core + 3.12.4 + test + + + + + + + false + src/main/resources + + + false + src/main/java + + ** + + + **/*.java + + + + false + src/main/python + + ** + + + + + + false + src/test/resources + + + false + src/test/java + + ** + + + **/*.java + + + + src/main/resources + ${project.basedir} + + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.1.0 + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + no-duplicate-declared-dependencies + + enforce + + + + + + + + + + + + org.codehaus.mojo + properties-maven-plugin + + + initialize + + read-project-properties + + + + build.properties + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.0 + + + package + + shade + + + myrobotlab + + true + myrobotlab-full + false + + + + + org.myrobotlab.service.Runtime + ${version} + ${version} + + ${build.number} + ${maven.build.timestamp} + ${agent.name} + ${user.name} + + + ${git.tags} + ${git.branch} + ${git.dirty} + ${git.remote.origin.url} + ${git.commit.id} + ${git.commit.id.abbrev} + ${git.commit.id.full} + ${git.commit.id.describe} + ${git.commit.id.describe-short} + ${git.commit.user.name} + ${git.commit.user.email} + + ${git.commit.time} + ${git.closest.tag.name} + ${git.closest.tag.commit.count} + ${git.build.user.name} + ${git.build.user.email} + ${git.build.time} + ${git.build.version} + + + + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + assembly.xml + + myrobotlab + false + + + + trigger-assembly + package + + single + + + + + + + true + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 11 + 11 + true + true + -parameters + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.4.3 + + + + pl.project13.maven + git-commit-id-plugin + 4.9.10 + + + initialize + get-the-git-infos + + revision + + + + + ${project.basedir}/.git + git + false + true + ${project.build.outputDirectory}/git.properties + + + false + false + -dirty + + + + + + maven-surefire-plugin + org.apache.maven.plugins + 2.22.2 + + -Djava.library.path=libraries/native -Djna.library.path=libraries/native + + **/*Test.java + + + **/integration/* + + + + + + + + org.apache.maven.plugins + maven-clean-plugin + 2.3 + + + + data/.myrobotlab + false + + + libraries + + ** + + false + + + data + + ** + + + + resource + + ** + + + + src/main/resources/resource/framework + + **/serviceData.json + + false + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.22.2 + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.1 + + + + + myrobotlab + http://myrobotlab.org + + + github + https://github.com/MyRobotLab/myrobotlab/issues + + diff --git a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java new file mode 100644 index 0000000000..536793fece --- /dev/null +++ b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java @@ -0,0 +1,118 @@ +package org.myrobotlab.ext.python; + +import org.bytedeco.javacpp.Loader; +import org.myrobotlab.framework.Platform; +import org.myrobotlab.io.FileIO; +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.process.SubprocessException; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.myrobotlab.framework.Status.error; +import static org.myrobotlab.io.FileIO.fs; + +public class PythonUtils { + private static Logger logger = LoggerFactory.getLogger(PythonUtils.class); + + public static final String SYSTEM_PYTHON_COMMAND = "python3"; + + + /** + * Ensures a virtual environment is setup at the + * desired location, creating it if it doesn't exist, + * and returning the absolute file path to the + * virtual environment Python executable. + * + * @param venv The location to setup the virtual environment + * @param useBundledPython Whether to use the bundled Python + * or the system python for setting up + * the virtual environment. + * @return The location of the virtual environment interpreter + */ + public static String setupVenv(String venv, boolean useBundledPython, List packages) throws IOException, InterruptedException { + String pythonCommand = (Platform.getLocalInstance().isWindows()) ? venv + fs + "Scripts" + fs + "python.exe" : venv + fs + "bin" + fs + "python"; + if (!FileIO.checkDir(venv)) { + // We don't have an initialized virtual environment, so lets make one + // and install our required packages + String hostPython = (useBundledPython) ? Loader.load(org.bytedeco.cpython.python.class) : SYSTEM_PYTHON_COMMAND; + ProcessBuilder installProcess; + int ret; + String venvLib = new File(hostPython).getParent() + fs + "lib" + fs + "venv" + fs + "scripts" + fs + "nt"; + if (Platform.getLocalInstance().isWindows()) { + // Super hacky workaround, venv works differently on Windows and requires these two + // files, but they are not distributed in bare-bones Python or in any pip packages. + // So we copy them where it expects, and it seems to work now + String containingDir = new File(hostPython).getParent(); + FileIO.copy(containingDir + fs + "python.exe", venvLib + fs + "python.exe"); + FileIO.copy(containingDir + fs + "pythonw.exe", venvLib + fs + "pythonw.exe"); + } + + installProcess = new ProcessBuilder(hostPython, "-m", "venv", venv); + ret = installProcess.inheritIO().start().waitFor(); + if (ret != 0) { + String message = String.format("Could not create virtual environment, subprocess returned %s. If on Windows, make sure there is a python.exe file in %s", ret, venvLib); + error(message); + throw new SubprocessException(message); + } + + + List command = new ArrayList<>(List.of(pythonCommand, "-m", "pip", "install")); + command.addAll(packages); + installProcess = new ProcessBuilder(command.toArray(new String[0])); + ret = installProcess.inheritIO().start().waitFor(); + if (ret != 0) { + String message = String.format("Could not install desired packages (%s)", packages); + error(message); + throw new SubprocessException(message); + } + + } + return new File(pythonCommand).getAbsolutePath(); + } + + /** + * Install a list of packages into the environment given by python. + * A new subprocess is spawned to perform the installation, output + * is echoed to this process's stdout/stderr. + *

+ * TODO add process gobbler to echo on logging system + * + * @param packages The list of packages to install. Must be findable by Pip + * @throws SubprocessException If an I/O error occurs running Pip. + */ + public static int installPipPackages(String python, List packages) { + ProcessBuilder builder = new ProcessBuilder(python, "-m", "pip", "install"); + List currCommand = builder.command(); + currCommand.addAll(packages); + try { + return builder.inheritIO().start().waitFor(); + } catch (InterruptedException | IOException e) { + throw new SubprocessException("Unable to install packages " + packages + " with Python command " + python, e); + } + + } + public static int runPythonScript(String python, File workingDirectory, String script, String... args) { + try { + return runPythonScriptAsync(python, workingDirectory, script, args).waitFor(); + } catch (InterruptedException e) { + throw new SubprocessException("Unable to run script " + script + " with Python command " + python, e); + } + } + + + public static Process runPythonScriptAsync(String python, File workingDirectory, String script, String... args) { + ProcessBuilder builder = new ProcessBuilder(python, script); + List currCommand = builder.command(); + currCommand.addAll(List.of(args)); + try { + return builder.inheritIO().directory(workingDirectory).start(); + } catch (IOException e) { + throw new SubprocessException("Unable to run script " + script + " with Python command " + python, e); + } + + } +} diff --git a/src/main/java/org/myrobotlab/framework/Registration.java b/src/main/java/org/myrobotlab/framework/Registration.java index 4f0b3070dc..5f43e9ffe7 100644 --- a/src/main/java/org/myrobotlab/framework/Registration.java +++ b/src/main/java/org/myrobotlab/framework/Registration.java @@ -72,7 +72,7 @@ public Registration(ServiceInterface service) { this.typeKey = service.getTypeKey(); // when this registration is re-broadcasted to remotes it will use this // serialization to init state - this.state = CodecUtils.toJson(service); + this.state = service.getSerializedState(); // if this is a local registration - need reference to service this.service = service; } diff --git a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java index 94a02c237e..bb4b6cfce8 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java @@ -4,4 +4,5 @@ public interface NameProvider { public String getName(); + } diff --git a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java index aad5e3a823..e3f2316fad 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java @@ -6,6 +6,8 @@ import java.util.Map; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.myrobotlab.codec.CodecUtils; import org.myrobotlab.framework.Inbox; import org.myrobotlab.framework.MRLListener; import org.myrobotlab.framework.MethodCache; @@ -178,10 +180,10 @@ public interface ServiceInterface extends ServiceQueue, LoggingSink, NameTypePro */ void setOrder(int creationCount); - String getId(); - String getFullName(); + String getId(); + void loadLocalizations(); void setLocale(String code); @@ -226,4 +228,9 @@ public interface ServiceInterface extends ServiceQueue, LoggingSink, NameTypePro * get all the subscriptions to this service */ public Map> getNotifyList(); + + @JsonIgnore + default String getSerializedState() { + return CodecUtils.toJson(this); + } } diff --git a/src/main/java/org/myrobotlab/io/FileIO.java b/src/main/java/org/myrobotlab/io/FileIO.java index 80d665e101..ea642737f9 100644 --- a/src/main/java/org/myrobotlab/io/FileIO.java +++ b/src/main/java/org/myrobotlab/io/FileIO.java @@ -392,6 +392,23 @@ static public final boolean extractResources() { return false; } + /** + * extractPythonServices extracts the entire python_services + * resource directory to the root directory, needed for all + * services written in Python. + * + * @return true if successful, false otherwise. + */ + static public boolean extractPythonServices() { + try { + extract(getRoot(), "python_services", null, false); + return true; + } catch (Exception e) { + Logging.logError(e); + } + return false; + } + /** * get configuration directory * diff --git a/src/main/java/org/myrobotlab/process/SubprocessException.java b/src/main/java/org/myrobotlab/process/SubprocessException.java new file mode 100644 index 0000000000..64fdd78b17 --- /dev/null +++ b/src/main/java/org/myrobotlab/process/SubprocessException.java @@ -0,0 +1,16 @@ +package org.myrobotlab.process; + +public class SubprocessException extends RuntimeException { + public SubprocessException(String message) { + super(message); + } + + public SubprocessException(Throwable throwable) { + super(throwable); + } + + public SubprocessException(String message, Throwable throwable) { + super(message, throwable); + } + +} diff --git a/src/main/java/org/myrobotlab/service/Py4j.java b/src/main/java/org/myrobotlab/service/Py4j.java index fd2755a733..1f7cf2e80d 100644 --- a/src/main/java/org/myrobotlab/service/Py4j.java +++ b/src/main/java/org/myrobotlab/service/Py4j.java @@ -1,20 +1,9 @@ package org.myrobotlab.service; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.bytedeco.javacpp.Loader; + import org.myrobotlab.codec.CodecUtils; +import org.myrobotlab.ext.python.PythonUtils; import org.myrobotlab.framework.Message; -import org.myrobotlab.framework.Platform; import org.myrobotlab.framework.Service; import org.myrobotlab.io.FileIO; import org.myrobotlab.io.StreamGobbler; @@ -25,11 +14,16 @@ import org.myrobotlab.service.data.Script; import org.myrobotlab.service.interfaces.Executor; import org.slf4j.Logger; - import py4j.GatewayServer; import py4j.GatewayServerListener; import py4j.Py4JServerConnection; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.*; + /** * * @@ -109,6 +103,7 @@ public void run() { try { exitCode = process.waitFor(); } catch (InterruptedException e) { + warn("Waiting for process was interrupted. Exit code cannot be known"); } warn("process %s terminated with exit code %d", process.toString(), exitCode); } @@ -140,7 +135,7 @@ public void run() { * executed or saved to the file system, or updatd in memory which the js * client does */ - protected HashMap openedScripts = new HashMap(); + protected HashMap openedScripts = new HashMap<>(); /** * client process and connectivity reference @@ -422,45 +417,11 @@ public void startPythonProcess() { // Script requires full name as first command line argument String[] pythonArgs = {getFullName()}; - // Build the command to start the Python process - ProcessBuilder processBuilder; - if (((Py4jConfig) config).useBundledPython) { - String venv = getDataDir() + fs + "venv"; - pythonCommand = (Platform.getLocalInstance().isWindows()) ? venv + fs + "Scripts" + fs + "python.exe" : venv + fs + "bin" + fs + "python"; - if (!FileIO.checkDir(venv)) { - // We don't have an initialized virtual environment, so lets make one - // and install our required packages - String python = Loader.load(org.bytedeco.cpython.python.class); - String venvLib = new File(python).getParent() + fs + "lib" + fs + "venv" + fs + "scripts" + fs + "nt"; - if (Platform.getLocalInstance().isWindows()) { - // Super hacky workaround, venv works differently on Windows and requires these two - // files, but they are not distributed in bare-bones Python or in any pip packages. - // So we copy them where it expects, and it seems to work now - FileIO.copy(getResourceDir() + fs + "python.exe", venvLib + fs + "python.exe"); - FileIO.copy(getResourceDir() + fs + "pythonw.exe", venvLib + fs + "pythonw.exe"); - } - ProcessBuilder installProcess = new ProcessBuilder(python, "-m", "venv", venv); - int ret = installProcess.inheritIO().start().waitFor(); - if (ret != 0) { - error("Could not create virtual environment, subprocess returned {}. If on Windows, make sure there is a python.exe file in {}", ret, venvLib); - return; - } - - installProcess = new ProcessBuilder(pythonCommand, "-m", "pip", "install", "py4j"); - ret = installProcess.inheritIO().start().waitFor(); - if (ret != 0) { - error("Could not install package, subprocess returned " + ret); - return; - } - } + String venv = getDataDir() + fs + "venv"; + pythonCommand = PythonUtils.setupVenv(venv, config.useBundledPython, List.of("py4j")); - // Virtual environment should exist, so lets use that python - } else { - // Just use the system python - pythonCommand = "python"; - } - processBuilder = new ProcessBuilder(pythonCommand, pythonScript); + ProcessBuilder processBuilder = new ProcessBuilder(pythonCommand, pythonScript); processBuilder.redirectErrorStream(true); processBuilder.command().addAll(List.of(pythonArgs)); @@ -513,7 +474,7 @@ public void installPipPackages(List packages) throws IOException { @Override public void startService() { super.startService(); - Py4jConfig c = (Py4jConfig)config; + Py4jConfig c = config; if (c.scriptRootDir == null) { c.scriptRootDir = new File(getDataInstanceDir()).getAbsolutePath(); } diff --git a/src/main/java/org/myrobotlab/service/Runtime.java b/src/main/java/org/myrobotlab/service/Runtime.java index f6000f2a2f..59c44397c0 100644 --- a/src/main/java/org/myrobotlab/service/Runtime.java +++ b/src/main/java/org/myrobotlab/service/Runtime.java @@ -20,6 +20,7 @@ import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -32,18 +33,23 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Queue; + +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; +import boofcv.alg.fiducial.qrcode.PackedBits8; import org.myrobotlab.codec.ClassUtil; import org.myrobotlab.codec.CodecUtils; import org.myrobotlab.codec.CodecUtils.ApiDescription; import org.myrobotlab.codec.ForeignProcessUtils; +import org.myrobotlab.ext.python.PythonUtils; + import org.myrobotlab.framework.CmdConfig; import org.myrobotlab.framework.CmdOptions; import org.myrobotlab.framework.DescribeQuery; @@ -66,10 +72,7 @@ import org.myrobotlab.framework.interfaces.MessageListener; import org.myrobotlab.framework.interfaces.NameProvider; import org.myrobotlab.framework.interfaces.ServiceInterface; -import org.myrobotlab.framework.repo.IvyWrapper; -import org.myrobotlab.framework.repo.Repo; -import org.myrobotlab.framework.repo.ServiceData; -import org.myrobotlab.framework.repo.ServiceDependency; +import org.myrobotlab.framework.repo.*; import org.myrobotlab.io.FileIO; import org.myrobotlab.logging.AppenderType; import org.myrobotlab.logging.LoggerFactory; @@ -88,11 +91,7 @@ import org.myrobotlab.service.config.ServiceConfig; import org.myrobotlab.service.data.Locale; import org.myrobotlab.service.data.ServiceTypeNameResults; -import org.myrobotlab.service.interfaces.ConnectionManager; -import org.myrobotlab.service.interfaces.Gateway; -import org.myrobotlab.service.interfaces.LocaleProvider; -import org.myrobotlab.service.interfaces.RemoteMessageHandler; -import org.myrobotlab.service.interfaces.ServiceLifeCyclePublisher; +import org.myrobotlab.service.interfaces.*; import org.myrobotlab.service.meta.abstracts.MetaData; import org.myrobotlab.string.StringUtil; import org.slf4j.Logger; @@ -124,7 +123,7 @@ * VAR OF RUNTIME ! * */ -public class Runtime extends Service implements MessageListener, ServiceLifeCyclePublisher, RemoteMessageHandler, ConnectionManager, Gateway, LocaleProvider { +public class Runtime extends Service implements MessageListener, ServiceLifeCyclePublisher, RemoteMessageHandler, ConnectionManager, Gateway, LocaleProvider, ServiceRunner { final static private long serialVersionUID = 1L; // FIXME - AVOID STATIC FIELDS !!! use .getInstance() to get the singleton @@ -135,6 +134,12 @@ public class Runtime extends Service implements MessageListener, */ static private final Map registry = new TreeMap<>(); + /** + * List of all services capable of running other services, + * even those outside of Java-land. + */ + private static final List serviceRunners = new CopyOnWriteArrayList<>(); + /** * A plan is a request to runtime to change the system. Typically its to ask * to start and configure new services. The master plan is an accumulation of @@ -245,7 +250,7 @@ public class Runtime extends Service implements MessageListener, */ transient private IvyWrapper repo = null; // was transient abstract Repo - transient private ServiceData serviceData = ServiceData.getLocalInstance(); + final transient private ServiceData serviceData = ServiceData.getLocalInstance(); /** * command line options @@ -316,6 +321,12 @@ public class Runtime extends Service implements MessageListener, */ protected Set startingServices = new HashSet<>(); + private static final String PYTHON_SERVICES_PATH = "python_services"; + + private static final String PYTHON_VENV_PATH = PYTHON_SERVICES_PATH + fs + "venv"; + + private String pythonCommand; + /** * Wraps {@link java.lang.Runtime#availableProcessors()}. * @@ -629,6 +640,30 @@ public void setAutoStart(boolean autoStart) throws IOException { invoke("getStartYml"); } + public void startPythonRuntime() { + PythonUtils.runPythonScriptAsync( + pythonCommand, + new File(PYTHON_SERVICES_PATH), + new File(PYTHON_SERVICES_PATH + fs + "mrl" + fs + "bootstrap.py").getAbsolutePath(), + Platform.getLocalInstance().getId() + "-python" + ); + } + + @Override + public List getSupportedLanguageKeys() { + return null; + } + + @Override + public List getAvailableServiceTypes() { + return Arrays.asList(getServiceNames()); + } + + @Override + public ServiceInterface startService(String name, String type) { + return Runtime.start(name, type); + } + /** * Framework owned method - core of creating a new service. This method will * create a service with the given name and of the given type. If the type @@ -694,7 +729,12 @@ static private synchronized ServiceInterface createService(String name, String t return null; } - String fullTypeName = CodecUtils.makeFullTypeName(type); + String fullTypeName; + if (ForeignProcessUtils.isForeignTypeKey(type) || type.contains(".")) { + fullTypeName = type; + } else { + fullTypeName = String.format("org.myrobotlab.service.%s", type); + } ServiceInterface si = Runtime.getService(fullName); if (si != null) { @@ -721,6 +761,32 @@ static private synchronized ServiceInterface createService(String name, String t return sw; } + if (ForeignProcessUtils.isForeignTypeKey(type)) { + String languageKey = ForeignProcessUtils.getLanguageId(type); + // Needed cause of lambda requiring effectively-final variables + String finalType = type; + List possibleRunners = serviceRunners.stream() + .filter(runner -> runner.getSupportedLanguageKeys().contains(languageKey)) + .filter(runner -> runner.getAvailableServiceTypes().contains(finalType)) + .collect(Collectors.toList()); + if (possibleRunners.isEmpty()) { + log.error("Cannot find a service runner to start service with type {}, all known runners: {}", type, serviceRunners); + return null; + } + + if (inId == null) { + return possibleRunners.get(0).startService(name, type); + } else { + Optional maybeRunner = possibleRunners.stream().filter(runner -> inId.equals(runner.getId())).findFirst(); + if (maybeRunner.isEmpty()) { + log.error("Cannot find compatible service runner with ID {}", inId); + return null; + } + return maybeRunner.get().startService(name, type); + + } + } + try { if (log.isDebugEnabled()) { @@ -907,6 +973,10 @@ public static Runtime getInstance() { Security.getInstance(); runtime.getRepo().addStatusPublisher(runtime); FileIO.extractResources(); + FileIO.extractPythonServices(); + + runtime.pythonCommand = PythonUtils.setupVenv(PYTHON_VENV_PATH, true, List.of("mrlpy")); + // protected services we don't want to remove when releasing a config runtime.startingServices.add("runtime"); runtime.startingServices.add("security"); @@ -1144,7 +1214,7 @@ public static Map getMethodMap(String inName) { * @return list of registrations */ synchronized public List getServiceList() { - return registry.values().stream().map(si -> new Registration(si.getId(), si.getName(), si.getTypeKey())).collect(Collectors.toList()); + return registry.values().stream().map(Registration::new).collect(Collectors.toList()); } // FIXME - scary function - returns private data @@ -1516,7 +1586,7 @@ static public void install(String serviceType) { * License - should be appropriately accepted or rejected by user * * @param serviceType - * the service tyype to install + * the service type to install * @param blocking * if this should block until done. * @@ -1534,8 +1604,24 @@ public void run() { try { if (serviceType == null) { r.getRepo().install(); + int returnCode = PythonUtils.runPythonScript( + r.pythonCommand, + new File("python_services"), + "python_services" + fs + "setup.py", + "install"); + if (returnCode != 0) { + r.error("Cannot install Python services, subprocess returned " + returnCode); + } } else { r.getRepo().install(serviceType); + int returnCode = PythonUtils.runPythonScript( + r.pythonCommand, + new File("python_services"), + "python_services" + fs + "setup.py", + "install", serviceType); + if (returnCode != 0) { + r.error("Cannot install Python services, subprocess returned " + returnCode); + } } } catch (Exception e) { r.error(e); @@ -1686,11 +1772,6 @@ public void onState(ServiceInterface updatedService) { registry.put(String.format("%s@%s", updatedService.getName(), updatedService.getId()), updatedService); } - public static synchronized Registration register(String id, String name, String typeKey, ArrayList interfaces) { - Registration proxy = new Registration(id, name, typeKey, interfaces); - register(proxy); - return proxy; - } /** * Registration is the process where a remote system sends detailed info @@ -1788,6 +1869,48 @@ public static synchronized Registration register(Registration registration) { } registry.put(fullname, registration.service); + if (registration.interfaces.contains(ServiceRunner.class.getName())) { + if (Runtime.class.isAssignableFrom(registration.service.getClass())) { + + // Might not be needed, I'm just not sure how calling these methods + // on an emulated Runtime like mrlpy's would work +// serviceRunners.add(new ServiceRunner() { +// @Override +// public String getName() { +// return null; +// } +// +// @Override +// public String getId() { +// return null; +// } +// +// @Override +// public List getSupportedLanguageKeys() { +// try { +// return (List) Runtime.get().sendBlocking(registration.getFullName(), "getSupportedLanguageKeys"); +// } catch (TimeoutException | InterruptedException e) { +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public List getAvailableServiceTypes() { +// try { +// return (List) Runtime.get().sendBlocking(registration.getFullName(), "getAvailableServiceTypes"); +// } catch (TimeoutException | InterruptedException e) { +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public ServiceInterface createService(String name, String type, String inId) { +// return null; +// } +// }); + } + serviceRunners.add((ServiceRunner) registration.service); + } if (runtime != null) { diff --git a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java new file mode 100644 index 0000000000..33b09b6cc6 --- /dev/null +++ b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java @@ -0,0 +1,27 @@ +package org.myrobotlab.service.interfaces; +import org.myrobotlab.framework.interfaces.ServiceInterface; + +import java.util.List; + +public interface ServiceRunner extends ServiceInterface { + + /** + * Get the runner's supported service programming language + * keys, such as {@code py} or {@code kt}. Used to route + * service creation to a Runtime that supports the language + * the service is written in. + * + * @return The language type keys the runner supports + */ + List getSupportedLanguageKeys(); + + /** + * Gets the list of service classes the runner + * can start and manage on its own. + * + * @return The list of available service types + */ + List getAvailableServiceTypes(); + + ServiceInterface startService(String name, String type); +} diff --git a/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java b/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java index bae28c5088..e06d87aa05 100644 --- a/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java +++ b/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java @@ -20,6 +20,14 @@ public MarySpeechMeta() { addDescription("Speech synthesis based on MaryTTS"); addDependency("de.dfki.mary", "marytts", "5.2", "pom"); +// exclude("org.slf4j", "slf4j-api"); +// exclude("commons-io", "commons-io"); +// exclude("log4j", "log4j"); +// exclude("commons-lang", "commons-lang"); +// exclude("com.google.guava", "guava"); +// exclude("org.apache.opennlp", "opennlp-tools"); +// exclude("org.apache.opennlp", "opennlp-maxent"); +// exclude("org.slf4j", "slf4j-log4j12"); // FIXME - use the following config file to generate the needed data for // loadVoice() // main config for voices @@ -39,15 +47,14 @@ public MarySpeechMeta() { exclude("org.slf4j", "slf4j-log4j12"); exclude("log4j", "log4j"); } + exclude("commons-io", "commons-io"); + exclude("log4j", "log4j"); + exclude("commons-lang", "commons-lang"); + exclude("com.google.guava", "guava"); + exclude("org.apache.opennlp", "opennlp-tools"); + exclude("org.apache.opennlp", "opennlp-maxent"); } - exclude("org.slf4j", "slf4j-api"); - exclude("commons-io", "commons-io"); - exclude("log4j", "log4j"); - exclude("commons-lang", "commons-lang"); - exclude("com.google.guava", "guava"); - exclude("org.apache.opennlp", "opennlp-tools"); - exclude("org.apache.opennlp", "opennlp-maxent"); - exclude("org.slf4j", "slf4j-log4j12"); + addDependency("org.apache.logging.log4j", "log4j-1.2-api", "2.12.1"); addDependency("org.apache.logging.log4j", "log4j-api", "2.12.1"); diff --git a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java index f21e872171..83a248a0b3 100644 --- a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java +++ b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java @@ -52,7 +52,16 @@ public RuntimeMeta() { addDependency("com.squareup.okhttp3", "okhttp", "3.9.0"); // force correct version of netty - needed for Vertx but not for Runtime ? - addDependency("io.netty", "netty-all", "4.1.82.Final"); + addDependency("io.netty", "netty-all", "4.1.82.Final"); + + // Used just as a Python exe redistributable. + // ABSOLUTELY NO JNI/JNA IS USED + addDependency("org.bytedeco", "cpython-platform", "3.10.8-1.5.8"); + addDependency("org.bytedeco", "cpython", "3.10.8-1.5.8"); + addDependency("org.bytedeco", "javacpp", "1.5.8"); + addDependency("org.bytedeco", "javacpp-platform", "1.5.8"); + +// addDependency("org.apache.commons", "commons-lang3", "3.3.2"); } diff --git a/src/main/python/README.md b/src/main/python/README.md new file mode 100644 index 0000000000..216d250448 --- /dev/null +++ b/src/main/python/README.md @@ -0,0 +1,25 @@ +## How Runtime starts a Python service +When the Java-side Runtime is instantiated, it extracts this python_services +directory, just like with resource. + +During install() when installing all, the virtual environment +is setup and configured, installing mrlpy and the python_services modules. + +A config option in Runtime's config sets whether to also start the python-side +runtime. If true, it first checks if the virtual environment is configured, +if so it starts a Python subprocess that execute's mrlpy's Runtime. + + +TODO replace explicit serviceRunners field with just finding services that implement +the interface when needed + +This Runtime connects to the Java-side and registers itself as a "ServiceRunner," +a new interface that denotes services able to start other services. + +When creating a service (calling Java-side Runtime.createService()) with +a foreign type (like py:mrlpy.services.TestService), Runtime will look +through all registered ServiceRunners to find any that can +start services with that language key. If it finds them, it will choose the +first one matching the desired ID to call createService() on. The service runner creates the service +on its side and then returns the constructed service, which Runtime then continues +to configure or start. diff --git a/src/main/python/python_services/mrl/bootstrap.py b/src/main/python/python_services/mrl/bootstrap.py new file mode 100644 index 0000000000..220a17b92b --- /dev/null +++ b/src/main/python/python_services/mrl/bootstrap.py @@ -0,0 +1,11 @@ +import logging +import sys + +from mrlpy import mcommand +from mrlpy.framework import runtime +from mrlpy.framework.runtime import Runtime +runtime.runtime_id = sys.argv[1] + +Runtime.init_runtime() +logging.basicConfig(level=logging.INFO, force=True) +mcommand.connect(id=sys.argv[1], daemon=False) diff --git a/src/main/python/python_services/mrl/services/TestService.py b/src/main/python/python_services/mrl/services/TestService.py new file mode 100644 index 0000000000..4357b1e7d9 --- /dev/null +++ b/src/main/python/python_services/mrl/services/TestService.py @@ -0,0 +1,5 @@ +from mrlpy.framework import Service + +class TestService(Service): + def __init__(self, name=""): + super(TestService, self).__init__(name) \ No newline at end of file diff --git a/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py b/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py new file mode 100644 index 0000000000..fec18313e4 --- /dev/null +++ b/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py @@ -0,0 +1,11 @@ +MetaData( + service_type="TestService", + available=False, + dependencies=( + "llamacpp-python=1.0.0", + "guidance=2.0.0" + + ), + description="A test service not for actual use", + is_cloud_service=True +) \ No newline at end of file diff --git a/src/main/python/python_services/mrl/services/meta/__init__.py b/src/main/python/python_services/mrl/services/meta/__init__.py new file mode 100644 index 0000000000..fd9e73498d --- /dev/null +++ b/src/main/python/python_services/mrl/services/meta/__init__.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass +from typing import Tuple + + +""" +Should be in mrlpy. +Need to add Runtime.install() +and an auto-import of mrlpy.services.meta.* +in the Runtime initializer +""" + +metadata_instances = dict() + +@dataclass +class MetaData(): + service_type: str + available: bool = True + dependencies: Tuple = () + description: str = "" + is_cloud_service: bool = False + + def __post_init__(self): + if self.service_type is None or self.service_type == "": + raise ValueError("Cannot have empty service type field") + if self.service_type in metadata_instances: + raise ValueError(f"Cannot redefine {self.service_type}'s metadata") + + metadata_instances.update({self.service_type: self}) + + + + diff --git a/src/main/resources/resource/framework/pom.xml.template b/src/main/resources/resource/framework/pom.xml.template index 8a72e3c9ef..929ff378e7 100644 --- a/src/main/resources/resource/framework/pom.xml.template +++ b/src/main/resources/resource/framework/pom.xml.template @@ -125,6 +125,13 @@ **/*.java + + false + src/main/python + + ** + +