Multi-board netlist generation (#487)

Add netlister functionality to generate multiple netlists for
sub-boards. The netlister can now return multiple netlists per design.

Changes netlisting where in subboard blocks, connections happen in both
the enclosing and sub-board scope. This is needed to handle connections
across the external facing boundary ports, external-scope blocks, and
internal-scope blocks.

Changes refdesing to have a single consistent numbering across all
sub-boards, instead of restarting every board at 1.

Author: ducky64

edg/BoardCompiler.py

@@ -3,6 +3,7 @@
 from contextlib import suppress
 from typing import Type, Optional, Tuple
 
+from . import edgir
 from .core import Block, ScalaCompiler, CompiledDesign, CompiledDesignExportTransform
 from .electronics_model.NetlistBackend import NetlistBackend  # imported separately b/c mypy confuses with the modules
 from .electronics_model.SvgPcbBackend import SvgPcbBackend
@@ -18,15 +19,16 @@ def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]
         assert os.path.isdir(target_dir), f"target_dir {target_dir} to compile_board must be directory"
 
         design_filename = os.path.join(target_dir, f"{target_name}.edg")
-        netlist_filename = os.path.join(target_dir, f"{target_name}.net")
+        netlist_filename_prefix = os.path.join(target_dir, f"{target_name}")
         bom_filename = os.path.join(target_dir, f"{target_name}.csv")
         svgpcb_filename = os.path.join(target_dir, f"{target_name}.svgpcb.js")
         compiled_json_filename = os.path.join(target_dir, f"{target_name}.compiled.json")
 
         with suppress(FileNotFoundError):
             os.remove(design_filename)
-        with suppress(FileNotFoundError):
-            os.remove(netlist_filename)
+        for filename in os.listdir(target_dir):
+            if filename.startswith(target_name) and filename.endswith(".net"):
+                os.remove(os.path.join(target_dir, filename))
         with suppress(FileNotFoundError):
             os.remove(bom_filename)
         with suppress(FileNotFoundError):
@@ -48,13 +50,21 @@ def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]
 
     netlist_all = NetlistBackend().run(compiled)
     bom_all = GenerateBom().run(compiled)
+    assert len(bom_all) == 1
     svgpcb_all = SvgPcbBackend().run(compiled)
+    assert len(svgpcb_all) == 1
     compiled_json = CompiledDesignExportTransform(compiled).transform()
-    assert len(netlist_all) == 1
 
     if target_dir_name is not None:
-        with open(netlist_filename, "w", encoding="utf-8") as net_file:
-            net_file.write(netlist_all[0][1])
+        for path, netlist in netlist_all:
+            path_str = edgir.local_path_to_str_list(path)
+            if not path_str:
+                net_filename = netlist_filename_prefix + ".net"
+            else:
+                net_filename = netlist_filename_prefix + "_" + "_".join(path_str) + ".net"
+
+            with open(net_filename, "w", encoding="utf-8") as net_file:
+                net_file.write(netlist)
 
         with open(bom_filename, "w", encoding="utf-8") as bom_file:
             bom_file.write(bom_all[0][1])

edg/abstract_parts/test_kicad_import_netlist.py

@@ -39,7 +39,7 @@ def __init__(self) -> None:
 
 class KiCadImportBlackboxTestCase(unittest.TestCase):
     def test_netlist(self) -> None:
-        net = NetlistTestCase.generate_net(
+        net = NetlistTestCase.generate_single_net(
             KiCadBlackboxTop,
             refinements=Refinements(
                 class_refinements=[

edg/electronics_model/BoardScopedTransform.py

@@ -15,9 +15,10 @@ class BoardScopedTransform(TransformUtil.Transform):
     def __init__(self, design: CompiledDesign) -> None:
         super().__init__()
         self._design = design
-        self._board_scopes: Dict[TransformUtil.Path, Optional[TransformUtil.Path]] = {
+        self._board_parent_scopes: Dict[TransformUtil.Path, Optional[TransformUtil.Path]] = {
             TransformUtil.Path.empty(): TransformUtil.Path.empty()
         }  # always initialized in parent
+        self._board_scopes: Dict[TransformUtil.Path, Optional[TransformUtil.Path]] = {}
 
     def visit_block_scoped(
         self, context: TransformUtil.TransformContext, scope: Optional[TransformUtil.Path], block: edgir.BlockTypes
@@ -36,7 +37,7 @@ def visit_linkarray_scoped(
 
     @override
     def visit_block(self, context: TransformContext, block: edgir.HierarchyBlock) -> None:
-        scope = self._board_scopes[context.path]
+        parent_scope = self._board_parent_scopes[context.path]
 
         if "fp_subboard" in block.meta.members.node:
             fp_external_blocks = self._design.get_value(context.path.to_tuple() + ("fp_external_blocks",))
@@ -48,15 +49,17 @@ def visit_block(self, context: TransformContext, block: edgir.HierarchyBlock) ->
                 internal_scope = context.path
         else:
             external_blocks = None
-            internal_scope = scope
+            internal_scope = parent_scope
+
+        self._board_scopes[context.path] = internal_scope
 
         for block_pair in block.blocks:
             if external_blocks is not None and block_pair.name not in external_blocks:
-                self._board_scopes[context.path.append_block(block_pair.name)] = internal_scope
+                self._board_parent_scopes[context.path.append_block(block_pair.name)] = internal_scope
             else:
-                self._board_scopes[context.path.append_block(block_pair.name)] = scope
+                self._board_parent_scopes[context.path.append_block(block_pair.name)] = parent_scope
 
-        self.visit_block_scoped(context, scope, block)
+        self.visit_block_scoped(context, internal_scope, block)
 
     @override
     def visit_link(self, context: TransformContext, link: edgir.Link) -> None:

edg/electronics_model/NetlistBackend.py

@@ -21,7 +21,8 @@ def run(self, design: CompiledDesign, args: Dict[str, str] = {}) -> List[Tuple[e
         else:
             raise ValueError(f"Invalid RefdesMode value {refdes_mode_arg}")
 
-        netlist = NetlistTransform(design).run()
-        netlist_string = kicad.generate_netlist(netlist, refdes_mode)
-
-        return [(edgir.LocalPath(), netlist_string)]
+        board_netlists = NetlistTransform(design).run()
+        return [
+            (netlist_path.to_local_path(), kicad.generate_netlist(netlist, refdes_mode))
+            for (netlist_path, netlist) in board_netlists.items()
+        ]

edg/electronics_model/NetlistGenerator.py

@@ -58,7 +58,6 @@ def empty(cls, path: TransformUtil.Path) -> "BoardScope":  # returns a fresh, em
         return BoardScope(path, {}, {}, {}, [])
 
 
-Scopes = Dict[TransformUtil.Path, Optional[BoardScope]]  # Block -> board scope (reference, aliased across entries)
 ClassPaths = Dict[
     TransformUtil.Path, List[edgir.LibraryPath]
 ]  # Path -> class names corresponding to shortened path name
@@ -82,8 +81,9 @@ def flatten_port(path: TransformUtil.Path, port: edgir.PortLike) -> Iterable[Tra
     def __init__(self, design: CompiledDesign):
         super().__init__(design)
 
-        self.all_scopes = [BoardScope.empty(TransformUtil.Path.empty())]  # list of unique scopes
-        self.scopes: Scopes = {TransformUtil.Path.empty(): self.all_scopes[0]}
+        self.scopes: Dict[TransformUtil.Path, Optional[BoardScope]] = {
+            TransformUtil.Path.empty(): BoardScope.empty(TransformUtil.Path.empty())
+        }  # board scope path to scope object
         self.class_paths: ClassPaths = {TransformUtil.Path.empty(): []}  # seed root
         self.path_traverse_order: List[TransformUtil.Path] = []
 
@@ -98,6 +98,16 @@ def process_blocklike(
         else:
             scope_obj = None
 
+        if isinstance(block, edgir.HierarchyBlock) and "fp_subboard" in block.meta.members.node:
+            # only valid for sub-board blocks, where some things happen in the parent scope
+            parent_scope = self._board_parent_scopes[path]
+            if parent_scope is not None:
+                parent_scope_obj: Optional[BoardScope] = self.scopes[parent_scope]
+            else:
+                parent_scope_obj = None
+        else:
+            parent_scope_obj = None
+
         if isinstance(block, edgir.HierarchyBlock):
             class_path = self.class_paths[path]
             for block_pair in block.blocks:
@@ -171,66 +181,85 @@ def process_blocklike(
                 scope_obj.edges.setdefault(src_path, [])  # make sure there is a port entry so single-pin nets are named
                 scope_obj.pins.setdefault(src_path, []).append(NetPin(path, pin_name))
 
-        for constraint_pair in block.constraints:
-            if scope_obj is not None:
+        # for blocks with mixed scope, connections happen in both scopes, since blocks may be in either
+        # this may cause inner connections to leach out into the parent scope, or
+        # result in extraneous connections in either scope,
+        # but is much simpler implementation-wise
+        all_scopes = []
+        if scope_obj is not None:
+            all_scopes.append(scope_obj)
+        if parent_scope_obj is not None and parent_scope_obj is not scope_obj:
+            all_scopes.append(parent_scope_obj)
+
+        if all_scopes:
+            for constraint_pair in block.constraints:
                 if constraint_pair.value.HasField("connected"):
-                    self.process_connected(path, block, scope_obj, constraint_pair.value.connected)
+                    self.process_connected(path, block, all_scopes, constraint_pair.value.connected)
                 elif constraint_pair.value.HasField("connectedArray"):
                     for expanded_connect in constraint_pair.value.connectedArray.expanded:
-                        self.process_connected(path, block, scope_obj, expanded_connect)
+                        self.process_connected(path, block, all_scopes, expanded_connect)
                 elif constraint_pair.value.HasField("exported"):
-                    self.process_exported(path, block, scope_obj, constraint_pair.value.exported)
+                    self.process_exported(path, block, all_scopes, constraint_pair.value.exported)
                 elif constraint_pair.value.HasField("exportedArray"):
                     for expanded_export in constraint_pair.value.exportedArray.expanded:
-                        self.process_exported(path, block, scope_obj, expanded_export)
+                        self.process_exported(path, block, all_scopes, expanded_export)
                 elif constraint_pair.value.HasField("exportedTunnel"):
-                    self.process_exported(path, block, scope_obj, constraint_pair.value.exportedTunnel)
+                    self.process_exported(path, block, all_scopes, constraint_pair.value.exportedTunnel)
 
     def process_connected(
-        self, path: TransformUtil.Path, current: edgir.EltTypes, scope: BoardScope, constraint: edgir.ConnectedExpr
+        self,
+        path: TransformUtil.Path,
+        current: edgir.EltTypes,
+        scopes: List[BoardScope],
+        constraint: edgir.ConnectedExpr,
     ) -> None:
         if constraint.expanded:
             assert len(constraint.expanded) == 1
-            self.process_connected(path, current, scope, constraint.expanded[0])
+            self.process_connected(path, current, scopes, constraint.expanded[0])
             return
         assert constraint.block_port.HasField("ref")
         assert constraint.link_port.HasField("ref")
         self.connect_ports(
-            scope, path.follow(constraint.block_port.ref, current), path.follow(constraint.link_port.ref, current)
+            scopes, path.follow(constraint.block_port.ref, current), path.follow(constraint.link_port.ref, current)
         )
 
     def process_exported(
-        self, path: TransformUtil.Path, current: edgir.EltTypes, scope: BoardScope, constraint: edgir.ExportedExpr
+        self,
+        path: TransformUtil.Path,
+        current: edgir.EltTypes,
+        scopes: List[BoardScope],
+        constraint: edgir.ExportedExpr,
     ) -> None:
         if constraint.expanded:
             assert len(constraint.expanded) == 1
-            self.process_exported(path, current, scope, constraint.expanded[0])
+            self.process_exported(path, current, scopes, constraint.expanded[0])
             return
         assert constraint.internal_block_port.HasField("ref")
         assert constraint.exterior_port.HasField("ref")
         self.connect_ports(
-            scope,
+            scopes,
             path.follow(constraint.internal_block_port.ref, current),
             path.follow(constraint.exterior_port.ref, current),
         )
 
     def connect_ports(
         self,
-        scope: BoardScope,
+        scopes: List[BoardScope],
         elt1: Tuple[TransformUtil.Path, edgir.EltTypes],
         elt2: Tuple[TransformUtil.Path, edgir.EltTypes],
     ) -> None:
         """Recursively connect ports, including containers and leaf ports. Net-ness is ignored here."""
         if isinstance(elt1[1], edgir.Port) and isinstance(elt2[1], edgir.Port):
-            scope.edges.setdefault(elt1[0], []).append(elt2[0])
-            scope.edges.setdefault(elt2[0], []).append(elt1[0])
+            for scope in scopes:
+                scope.edges.setdefault(elt1[0], []).append(elt2[0])
+                scope.edges.setdefault(elt2[0], []).append(elt1[0])
 
             elt1_names = list(map(lambda pair: pair.name, elt1[1].ports))
             elt2_names = list(map(lambda pair: pair.name, elt2[1].ports))
             assert elt1_names == elt2_names, f"mismatched port sub-ports in types {elt1}, {elt2}"
             for key in elt2_names:
                 self.connect_ports(
-                    scope,
+                    scopes,
                     (elt1[0].append_port(key), edgir.resolve_portlike(edgir.pair_get(elt1[1].ports, key))),
                     (elt2[0].append_port(key), edgir.resolve_portlike(edgir.pair_get(elt2[1].ports, key))),
                 )
@@ -371,11 +400,10 @@ def port_ignored_paths(path: TransformUtil.Path) -> bool:  # ignore link ports f
 
         return Netlist(netlist_footprints, netlist_nets)
 
-    def run(self) -> Netlist:
+    def run(self) -> Dict[TransformUtil.Path, Netlist]:
         self.transform_design(self._design.design)
 
-        assert len(self.all_scopes) == 1, "TODO: support multiple boards"
-        return self.scope_to_netlist(self.all_scopes[0])
+        return {path: self.scope_to_netlist(scope) for path, scope in self.scopes.items() if scope is not None}
 
 
 class PathShortener:

edg/electronics_model/RefdesRefinementPass.py

@@ -30,7 +30,7 @@ def __init__(self, design: CompiledDesign):
             self.board_refdes_prefix = board_refdes_prefix
 
         self.block_refdes_list: List[Tuple[TransformUtil.Path, str]] = []  # populated in traversal order
-        self.refdes_last: Dict[Tuple[TransformUtil.Path, str], int] = {}  # (scope, prefix) -> num
+        self.refdes_last: Dict[str, int] = {}  # prefix -> num
 
     @override
     def visit_block_scoped(
@@ -40,8 +40,8 @@ def visit_block_scoped(
             refdes_prefix = self._design.get_value(context.path.to_tuple() + ("fp_refdes_prefix",))
             assert isinstance(refdes_prefix, str)
 
-            refdes_id = self.refdes_last.get((scope, refdes_prefix), 0) + 1
-            self.refdes_last[(scope, refdes_prefix)] = refdes_id
+            refdes_id = self.refdes_last.get(refdes_prefix, 0) + 1
+            self.refdes_last[refdes_prefix] = refdes_id
             self.block_refdes_list.append((context.path, self.board_refdes_prefix + refdes_prefix + str(refdes_id)))
 
     def run(self) -> List[Tuple[TransformUtil.Path, str]]:

edg/electronics_model/SvgPcbBackend.py

@@ -190,7 +190,7 @@ def run(self) -> List[SvgPcbGeneratedBlock]:
 class SvgPcbBackend(BaseBackend):
     @override
     def run(self, design: CompiledDesign, args: Dict[str, str] = {}) -> List[Tuple[edgir.LocalPath, str]]:
-        netlist = NetlistTransform(design).run()
+        netlist = NetlistTransform(design).run()[TransformUtil.Path.empty()]  # only generate for top-level board
         result = self._generate(design, netlist)
         return [(edgir.LocalPath(), result)]
 
@@ -213,7 +213,7 @@ def filter_blocks_by_pathname(
         svgpcb_block_bboxes = [BlackBoxBlock(block.path, block.bbox) for block in svgpcb_blocks]
 
         # handle footprints
-        netlist = NetlistTransform(design).run()
+        netlist = NetlistTransform(design).run()[TransformUtil.Path.empty()]  # only generate for top-level board
         svgpcb_block_prefixes = [block.path.to_tuple() for block in svgpcb_blocks]
         other_blocks = filter_blocks_by_pathname(netlist.blocks, svgpcb_block_prefixes)
         arranged_blocks = arrange_blocks(other_blocks, svgpcb_block_bboxes)

edg/electronics_model/test_bundle_netlist.py

@@ -123,7 +123,7 @@ def contents(self) -> None:
 
 class BundleNetlistTestCase(unittest.TestCase):
     def test_spi_netlist(self) -> None:
-        net = NetlistTestCase.generate_net(TestSpiCircuit)
+        net = NetlistTestCase.generate_single_net(TestSpiCircuit)
 
         self.assertIn(
             Net(
@@ -237,7 +237,7 @@ def test_spi_netlist(self) -> None:
         )
 
     def test_uart_netlist(self) -> None:
-        net = NetlistTestCase.generate_net(TestUartCircuit)
+        net = NetlistTestCase.generate_single_net(TestUartCircuit)
 
         self.assertIn(
             Net(
@@ -285,7 +285,7 @@ def test_uart_netlist(self) -> None:
         )
 
     def test_can_netlist(self) -> None:
-        net = NetlistTestCase.generate_net(TestCanCircuit)
+        net = NetlistTestCase.generate_single_net(TestCanCircuit)
 
         self.assertIn(
             Net(

edg/electronics_model/test_multipack_netlist.py

@@ -78,7 +78,7 @@ def multipack(self) -> None:
 
 class MultipackNetlistTestCase(unittest.TestCase):
     def test_packed_netlist(self) -> None:
-        net = NetlistTestCase.generate_net(TestPackedDevices)
+        net = NetlistTestCase.generate_single_net(TestPackedDevices)
 
         self.assertIn(
             Net(
@@ -152,4 +152,4 @@ def test_invalid_netlist(self) -> None:
         from .NetlistGenerator import InvalidPackingException
 
         with self.assertRaises(InvalidPackingException):
-            NetlistTestCase.generate_net(TestInvalidPackedDevices)
+            NetlistTestCase.generate_single_net(TestInvalidPackedDevices)