Python packaging defs (#247)

- Add pyproject.toml
- Refactor paths for packaging:
  - precompiled compiler now in edg_core/resources
- HDL interface server in its own package, to be runnable with `python
-m`
- Update documentation
- Package uploaded to pip

Author: ducky64

.gitignore

@@ -23,6 +23,11 @@ BlinkyExample/
 
 Pipfile.lock
 
+# Python packaging
+build
+edg.egg-info
+dist
+
 # mypy
 .mypy_cache/
 .dmypy.json

LibraryDump.py

@@ -0,0 +1 @@
+include edg/edg-compiler-precompiled.jar

MANIFEST.in

@@ -14,8 +14,8 @@ import java.io.{File, PrintWriter, StringWriter}
 
 // a PythonInterface that uses the on-event hooks to forward stderr and stdout
 // without this, the compiler can freeze on large stdout/stderr data, possibly because of queue sizing
-class ForwardingPythonInterface(serverFile: File)
-    extends PythonInterface(serverFile) {
+class ForwardingPythonInterface(serverFile: Option[File], pythonPaths: Seq[String])
+    extends PythonInterface(serverFile, pythonPaths) {
   def forwardProcessOutput(): Unit = {
     StreamUtils.forAvailable(processOutputStream) { data =>
       System.out.print(new String(data))
@@ -79,7 +79,9 @@ object CompilerServerMain {
   }
 
   def main(args: Array[String]): Unit = {
-    val pyIf = new ForwardingPythonInterface(new File("HdlInterfaceService.py"))  // use relative path
+    val hdlServerOption = PythonInterface.serverFileOption(None)  // local relative path
+    hdlServerOption.foreach { serverFile => println(s"Using local $serverFile") }
+    val pyIf = new ForwardingPythonInterface(hdlServerOption, Seq(new File(".").getAbsolutePath))
     val pyLib = new PythonInterfaceLibrary()
     pyLib.withPythonInterface(pyIf) {
       while (true) {

compiler/src/main/scala/edg/compiler/CompilerServerMain.scala

@@ -21,12 +21,20 @@ object ProtobufStdioSubprocess {
 }
 
 
-class ProtobufStdioSubprocess
-    [RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage](
+class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage](
     responseType: scalapb.GeneratedMessageCompanion[ResponseType],
-    args: Seq[String]) {
+    pythonPaths: Seq[String], args: Seq[String]) {
   protected val process: Either[Process, Throwable] = try {
-    Left(new ProcessBuilder(args: _*).start())
+    val processBuilder = new ProcessBuilder(args: _*)
+    if (pythonPaths.nonEmpty) {
+      val env = processBuilder.environment()
+      val pythonPathString = pythonPaths.mkString(";")
+      Option(env.get("PYTHONPATH")) match {  // merge existing PYTHONPATH if exists
+        case None => env.put("PYTHONPATH", pythonPathString)
+        case Some(envPythonPath) => env.put("PYTHONPATH", envPythonPath + ";" + pythonPathString)
+      }
+    }
+    Left(processBuilder.start())
   } catch {
     case e: Throwable => Right(e)  // if it fails store the exception to be thrown when we can 
   }
@@ -113,14 +121,36 @@ class ProtobufStdioSubprocess
 }
 
 
+object PythonInterface {
+  private val kHdlServerFilePath = "edg_hdl_server/__main__.py"
+  // returns the HDL server Python script if it exists locally, otherwise returns None.
+  def serverFileOption(root: Option[File] = None): Option[File] = {
+    val hdlServerFile = root match {
+      case Some(root) => new File(root, kHdlServerFilePath)
+      case None => new File(kHdlServerFilePath)
+    }
+    if (hdlServerFile.exists()) {
+      Some(hdlServerFile)
+    } else {
+      None
+    }
+  }
+}
+
+
 /** An interface to the Python HDL elaborator, which reads in Python HDL code and (partially) compiles
   * them down to IR.
   * The underlying Python HDL should not change while this is open. This will not reload updated Python HDL files.
+  *
+  * If the serverFile is specified, run that; otherwise use "python -m edg_hdl_server" for the global package.
   */
-class PythonInterface(serverFile: File, pythonInterpreter: String = "python") {
+class PythonInterface(serverFile: Option[File], pythonPaths: Seq[String], pythonInterpreter: String = "python") {
+  val command = serverFile match {  // -u for unbuffered mode
+    case Some(serverFile) => Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath)
+    case None => Seq(pythonInterpreter, "-u", "-m", "edg_hdl_server")
+  }
   protected val process = new ProtobufStdioSubprocess[edgrpc.HdlRequest, edgrpc.HdlResponse](
-    edgrpc.HdlResponse,
-    Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath))  // in unbuffered mode
+    edgrpc.HdlResponse, pythonPaths, command)
   val processOutputStream: InputStream = process.outputStream
   val processErrorStream: InputStream = process.errorStream
 

compiler/src/main/scala/edg/compiler/PythonInterface.scala

@@ -13,7 +13,8 @@ class PythonInterfaceTest extends AnyFlatSpec {
     val compiledDir = new File(getClass.getResource("").getPath)
     // above returns compiler/target/scala-2.xx/test-classes/edg/compiler, get the root repo dir
     val repoDir = compiledDir.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile
-    val pyIf = new PythonInterface(new File(repoDir, "HdlInterfaceService.py"))
+    val pyIf = new PythonInterface(Some(new File(repoDir, "edg_hdl_server/__main__.py")),
+      Seq(repoDir.getAbsolutePath))
     pyIf.indexModule("edg_core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]])
     pyIf.shutdown() should equal(0)
   }

compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala

@@ -50,8 +50,8 @@ def append_values(self, values: List[Tuple[edgir.LocalPath, edgir.ValueLit]]):
 
 
 class ScalaCompilerInstance:
-  PRECOMPIED_RELPATH = "compiler/edg-compiler-precompiled.jar"
-  DEV_RELPATH = "compiler/target/scala-2.13/edg-compiler-assembly-0.1-SNAPSHOT.jar"
+  DEV_RELPATH = "../compiler/target/scala-2.13/edg-compiler-assembly-0.1-SNAPSHOT.jar"
+  PRECOMPIED_RELPATH = "resources/edg-compiler-precompiled.jar"
 
   def __init__(self, *, suppress_stderr: bool = False):
     self.process: Optional[Any] = None
@@ -61,13 +61,15 @@ def __init__(self, *, suppress_stderr: bool = False):
 
   def check_started(self) -> None:
     if self.process is None:
-      if os.path.exists(self.DEV_RELPATH):
-        jar_path = self.DEV_RELPATH
+      dev_path = os.path.join(os.path.dirname(__file__), self.DEV_RELPATH)
+      precompiled_path = os.path.join(os.path.dirname(__file__), self.PRECOMPIED_RELPATH)
+      if os.path.exists(dev_path):
+        jar_path = dev_path
         print("Using development JAR")
-      elif os.path.exists(self.PRECOMPIED_RELPATH):
-        jar_path = self.PRECOMPIED_RELPATH
+      elif os.path.exists(precompiled_path):
+        jar_path = precompiled_path
       else:
-        raise ValueError("No EDG Compiler JAR found")
+        raise ValueError(f"No EDG Compiler JAR found")
 
       self.process = subprocess.Popen(
         ['java', '-jar', jar_path],

edg_core/ScalaCompilerInterface.py

@@ -10,6 +10,9 @@
 from edg_core.Core import NonLibraryProperty
 
 
+EDG_PROTO_VERSION = 1
+
+
 class LibraryElementIndexer:
   """Indexer for libraries, recursively searches modules and their LibraryElements."""
   def __init__(self):
@@ -24,8 +27,8 @@ def index_module(self, module: ModuleType) -> Set[Type[LibraryElement]]:
   def _search_module(self, module: ModuleType) -> None:
     # avoid repeated work and re-indexing modules
     if (module.__name__ in sys.builtin_module_names
-        or not hasattr(module, '__file__')  # apparently load six.moves breaks
-        or module in self.seen_modules):
+            or not hasattr(module, '__file__')  # apparently load six.moves breaks
+            or module in self.seen_modules):
       return
     self.seen_modules.add(module)
 
@@ -34,8 +37,8 @@ def _search_module(self, module: ModuleType) -> None:
         self._search_module(member)
 
       if inspect.isclass(member) and issubclass(member, LibraryElement) and not issubclass(member, DesignTop) \
-          and member not in self.seen_elements \
-          and (member, NonLibraryProperty) not in member._elt_properties:  # process elements
+              and member not in self.seen_elements \
+              and (member, NonLibraryProperty) not in member._elt_properties:  # process elements
         self.seen_elements.add(member)
 
         for mro in member.mro():
@@ -67,7 +70,7 @@ def elaborate_class(elt_cls: Type[LibraryElementType]) -> Tuple[LibraryElementTy
 
 LibraryClassType = TypeVar('LibraryClassType')
 def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[LibraryClassType]) -> \
-    Type[LibraryClassType]:
+        Type[LibraryClassType]:
   elt_split = elt.target.name.split('.')
   elt_module = importlib.import_module('.'.join(elt_split[:-1]))
   assert inspect.ismodule(elt_module)
@@ -76,9 +79,7 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library
   return cls
 
 
-# In some cases stdout seems to buffer excessively, in which case starting python with -u seems to work
-# https://stackoverflow.com/a/35467658/5875811
-if __name__ == '__main__':
+def run_server():
   stdin_deserializer = BufferDeserializer(edgrpc.HdlRequest, sys.stdin.buffer)
   stdout_serializer = BufferSerializer[edgrpc.HdlResponse](sys.stdout.buffer)
 
@@ -137,6 +138,8 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library
           response_result = response.run_backend.results.add()
           response_result.path.CopyFrom(path)
           response_result.text = backend_result
+      elif request.HasField('get_proto_version'):
+        response.get_proto_version = EDG_PROTO_VERSION
       else:
         raise RuntimeError(f"Unknown request {request}")
     except BaseException as e:
@@ -149,3 +152,9 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library
 
     sys.stdout.buffer.write(stdin_deserializer.read_stdout())
     stdout_serializer.write(response)
+
+
+# In some cases stdout seems to buffer excessively, in which case starting python with -u seems to work
+# https://stackoverflow.com/a/35467658/5875811
+if __name__ == '__main__':
+  run_server()