Add docs to IR and JSON export (#484)

Structurally, this adds docs as a metadata field. indexed by the port or
param name, with the block docstring itself being the empty string as
the root.

For json export, this only applies to blocks, not links.

This changes the export format for port-arrays to allow a doc field.
This also excludes all None (null) values from the export to reduce
clutter but provides a default for importability.

Author: ducky64

edg/BoardCompiler.py

@@ -65,7 +65,9 @@ def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]
 
         with open(compiled_json_filename, "w", encoding="utf-8") as compiled_json_file:
             compiled_json_file.write(
-                CompiledDesignExportTransform.postprocess_serialized_json(compiled_json.model_dump_json(indent=2))
+                CompiledDesignExportTransform.postprocess_serialized_json(
+                    compiled_json.model_dump_json(indent=2, exclude_none=True)
+                )
             )
 
     return compiled

edg/core/Blocks.py

@@ -345,6 +345,19 @@ def _elaborated_def_to_proto(self) -> edgir.BlockLikeTypes:
 
         return self._def_to_proto()
 
+    def _add_doc_metadata(self) -> None:
+        """Adds docstrings to metadata"""
+        metadata_dict: Dict[str, str] = {}
+        if self.__doc__:
+            metadata_dict[""] = inspect.cleandoc(self.__doc__)
+        for name, param in self._parameters.items():
+            if param in self._param_docs:
+                metadata_dict[name] = self._param_docs[param]
+        for name, port in self._ports.items():
+            if port in self._port_docs:
+                metadata_dict[name] = self._port_docs[port]
+        self._docs = self.Metadata(metadata_dict)
+
     def _populate_def_proto_block_base(self, pb: edgir.BlockLikeTypes) -> None:
         """Populates the structural parts of a block proto: parameters, ports, superclasses"""
         assert (
@@ -392,6 +405,7 @@ def _populate_def_proto_block_base(self, pb: edgir.BlockLikeTypes) -> None:
 
         self._constraints.finalize()  # needed for source locator generation
 
+        self._add_doc_metadata()
         self._populate_metadata(pb.meta, self._metadata, ref_map)
 
     def _populate_def_proto_port_init(self, pb: edgir.BlockLikeTypes, ref_map: Refable.RefMapType) -> None:

edg/core/CompiledDesignExport.py

@@ -18,19 +18,23 @@
 class CompiledParam(BaseModel):
     # this is minimalistic so the output json is more compact
     type: str
-    value: Optional[Any]  # solved value, if available
+    value: Optional[Any] = None  # solved value, if available
+    value_excluded: Optional[bool] = None  # true if value excluded, None otherwise to skip the field
+    doc: Optional[str] = None  # doc specified by its parent block, if available
 
 
-class CompiledPortArray(RootModel[Dict[str, Union["CompiledPort", "CompiledPortArray"]]]):
-    pass
+class CompiledPortArray(BaseModel):
+    doc: Optional[str] = None  # doc specified by its parent block, if available
+    elts: Dict[str, Union["CompiledPort", "CompiledPortArray"]]
 
 
 class CompiledPort(BaseModel):
     path: PathType  # provide the full path to allow searchability
     cls: str  # self class
     # path of connected port, if connected
     # for block ports, this is the link, if connected to one
-    connected_path: Optional[Union[PathType, List[PathType]]]
+    connected_path: Optional[Union[PathType, List[PathType]]] = None
+    doc: Optional[str] = None  # doc specified by its parent block, if available
     # note, link ports do not have parameters (they inherit parameters from connected ports and are deduplicated here)
     params: Dict[str, CompiledParam]
     ports: Dict[str, Union["CompiledPort", CompiledPortArray]]
@@ -48,6 +52,7 @@ class CompiledBlock(BaseModel):
     path: PathType  # provide the full path to allow searchability
     cls: str  # self class
     superclasses: List[str]  # all superclasses
+    doc: Optional[str] = None  # docstring, if available
     params: Dict[str, CompiledParam]
     ports: Dict[str, Union[CompiledPortArray, CompiledPort]]
     blocks: Dict[str, "CompiledBlock"]  # sub-blocks
@@ -108,14 +113,10 @@ def _param_value_to_json(cls, value: Any) -> Any:
 
     def _param_to_compiled(self, path: Path, elt: edgir.ValInit) -> CompiledParam:
         if path.params[-1] in self._EXCLUDED_PARAM_VALUES:
-            value: Optional[Any] = "<excluded>"
+            return CompiledParam(type=self._param_to_type(elt), value_excluded=True)
         else:
             value = self._param_value_to_json(self._design.get_value(path.to_local_path()))
-
-        return CompiledParam(
-            type=self._param_to_type(elt),
-            value=value,
-        )
+            return CompiledParam(type=self._param_to_type(elt), value=value)
 
     @override
     def transform_block(
@@ -126,16 +127,40 @@ def transform_block(
         blocks: Mapping[str, CompiledBlock],
         links: Mapping[str, CompiledLink],
     ) -> CompiledBlock:
+        ports_dict = dict(ports)
+        params_dict = {
+            param_pair.name: self._param_to_compiled(context.path.append_param(param_pair.name), param_pair.value)
+            for param_pair in elt.params
+        }
+
+        meta_docs = elt.meta.members.node.get("_docs")
+        if meta_docs is not None:
+            block_doc = meta_docs.members.node.get("")
+            if block_doc is not None:
+                block_doc = block_doc.text_leaf
+
+            for param_name, param_value in params_dict.items():
+                if param_name in meta_docs.members.node:
+                    param_doc = meta_docs.members.node.get(param_name)
+                    if param_doc is not None:
+                        param_value.doc = param_doc.text_leaf
+
+            for port_name, port_value in ports_dict.items():
+                if port_name in meta_docs.members.node:
+                    port_doc = meta_docs.members.node.get(port_name)
+                    if port_doc is not None:
+                        port_value.doc = port_doc.text_leaf
+        else:
+            block_doc = None
+
         return CompiledBlock(
             path=self._path_to_path(context.path),
             cls=self._libpath_to_str(elt.self_class),
             superclasses=[self._libpath_to_str(cls) for cls in elt.superclasses]
             + [self._libpath_to_str(cls) for cls in elt.super_superclasses],
-            params={
-                param_pair.name: self._param_to_compiled(context.path.append_param(param_pair.name), param_pair.value)
-                for param_pair in elt.params
-            },
-            ports=dict(ports),
+            doc=block_doc,
+            params=params_dict,
+            ports=ports_dict,
             blocks=dict(blocks),
             links=dict(links),
         )
@@ -179,7 +204,7 @@ def transform_port(
                 ports=dict(ports),
             )
         elif isinstance(elt, edgir.PortArray):
-            return CompiledPortArray(dict(ports))
+            return CompiledPortArray(elts=dict(ports))
         else:
             raise ValueError(f"unknown port type {type(elt)}")
 
@@ -219,8 +244,13 @@ def postprocess_serialized_json(json_str: str) -> str:
             json_str,
         )
         json_str = re.sub(
-            r'\{\s*"type":\s*"(\S+)",\s*"value":\s*(.+)\s*\}',
-            lambda m: rf'{{ "type": "{m.group(1)}", "value": {m.group(2)} }}',
+            r'\{\s*("type":\s*"\S+"),\s*("value":\s*.+)\s*\}',
+            lambda m: rf"{{ {m.group(1)}, {m.group(2)} }}",
+            json_str,
+        )
+        json_str = re.sub(
+            r'\{\s*("type":\s*"\S+"),\s*("value":\s*.+),\s*("doc":\s*.+)\s*\}',
+            lambda m: rf"{{ {m.group(1)}, {m.group(2)}, {m.group(3)} }}",
             json_str,
         )
         return json_str

edg/core/test_block.py

@@ -31,12 +31,14 @@ class TestBlockSecondNonLibrary(TestBlockSecondSub):  # test that it can skip th
 
 
 class TestBlock(TestBlockBase, TestBlockSecondNonLibrary):
+    """Test docstring"""
+
     def __init__(self) -> None:
         super().__init__()
-        self.range_init = self.Parameter(RangeExpr((-4.2, -1.3)))
+        self.range_init = self.Parameter(RangeExpr((-4.2, -1.3)), doc="range with initializer")
         self.array_init = self.Parameter(ArrayBoolExpr([False, True, False]))
         self.array_empty = self.Parameter(ArrayStringExpr([]))
-        self.port_lit = self.Port(TestPortBase(117), optional=True)
+        self.port_lit = self.Port(TestPortBase(117), optional=True, doc="port with initializer")
 
 
 class BlockBaseProtoTestCase(unittest.TestCase):
@@ -153,3 +155,10 @@ def test_param_init(self) -> None:
         expected_assign.assign.src.array.SetInParent()
         self.assertEqual(self.pb.constraints[5].name, "(init)array_empty")
         self.assertEqual(self.pb.constraints[5].value, expected_assign)
+
+    def test_docs(self) -> None:
+        self.assertEqual(self.pb.meta.members.node["_docs"].members.node[""].text_leaf, "Test docstring")
+        self.assertEqual(
+            self.pb.meta.members.node["_docs"].members.node["range_init"].text_leaf, "range with initializer"
+        )
+        self.assertEqual(self.pb.meta.members.node["_docs"].members.node["port_lit"].text_leaf, "port with initializer")

edg/core/test_common.py

@@ -35,9 +35,11 @@ class TestPortSink(TestPortBase):
 
 
 class TestBlockSink(Block):
+    """Sink block"""
+
     def __init__(self) -> None:
         super().__init__()
-        self.sink = self.Port(TestPortSink(), optional=True)
+        self.sink = self.Port(TestPortSink(), optional=True, doc="Sink port")
 
 
 ImplicitSink = PortTag(TestPortSink)
@@ -50,9 +52,11 @@ def __init__(self) -> None:
 
 
 class TestBlockSource(Block):
+    """Source block"""
+
     def __init__(self) -> None:
         super().__init__()
-        self.source = self.Port(TestPortSource(), optional=True)
+        self.source = self.Port(TestPortSource(), optional=True, doc="Source port")
 
 
 def problem_name_from_module_file(file: str) -> str:

edg/core/test_compiled_design_export.py

@@ -12,12 +12,18 @@ def test_hierarchy(self) -> None:
         compiled = ScalaCompiler.compile(TopHierarchyBlock)
         result = CompiledDesignExportTransform(compiled).transform()
         self.assertEqual(result.blocks["source"].cls, "edg.core.test_common.TestBlockSource")
+        self.assertEqual(result.blocks["source"].doc, "Source block")
         self.assertEqual(result.blocks["sink1"].cls, "edg.core.test_common.TestBlockSink")
+        self.assertEqual(result.blocks["sink1"].doc, "Sink block")
         self.assertEqual(result.blocks["sink2"].cls, "edg.core.test_common.TestBlockSink")
+        self.assertEqual(result.blocks["sink2"].doc, "Sink block")
         self.assertEqual(result.links["test_net"].cls, "edg.core.test_common.TestLink")
         self.assertEqual(cast(CompiledPort, result.blocks["source"].ports["source"]).connected_path, "test_net.source")
+        self.assertEqual(cast(CompiledPort, result.blocks["source"].ports["source"]).doc, "Source port")
         self.assertEqual(cast(CompiledPort, result.blocks["sink1"].ports["sink"]).connected_path, "test_net.sinks.0")
+        self.assertEqual(cast(CompiledPort, result.blocks["sink1"].ports["sink"]).doc, "Sink port")
         self.assertEqual(cast(CompiledPort, result.blocks["sink2"].ports["sink"]).connected_path, "test_net.sinks.1")
+        self.assertEqual(cast(CompiledPort, result.blocks["sink2"].ports["sink"]).doc, "Sink port")
 
     def test_param_values(self) -> None:
         from .test_simple_expr_eval import TestEvalReductionBlock