small bug fixes and commenting improvements

Author: robertmuth

BE/Base/cfg.py

@@ -270,7 +270,8 @@ def FunAddUnconditionalBranches(fun: ir.Fun):
             if n + 1 == len(fun.bbls) or fun.bbls[n + 1] != succ:
                 bbl.inss.append(ir.Ins(o.BRA, [succ], False))
             continue
-        # more complex case - we have two successors, we need to check if one of them is the fallthrough and if not we need to add a branch to the fallthrough
+        # more complex case - we have two successors, we need to check if one of them
+        # is the fallthrough and if not we need to add a branch to the fallthrough
         assert len(bbl.edge_out) == 2
         cond_bra = bbl.inss[-1]
         assert cond_bra.opcode.kind is o.OPC_KIND.COND_BRA, (

FE/lexer_tab.py

@@ -28,7 +28,10 @@
 NODE = list[int]
 TRIE = list[list[int]]
 
-
+# The integer in a trie nodes are encoded as follows:
+# 0..len(trie) - 1: index of the next node in the
+# TK_KIND_OFFSET - TK_KIND_OFFSET_FOR_LOOK_AHEAD: the token kind is not longer part of the kw
+# TK_KIND_OFFSET_FOR_LOOK_AHEAD -: the token kind if the current node is a terminal
 TK_KIND_OFFSET = 1000
 TK_KIND_OFFSET_FOR_LOOK_AHEAD = 2000
 
@@ -121,19 +124,21 @@ def rewrite(t: NODE):
     return out
 
 
-def FindInTrie(trie: TRIE, s: str) -> tuple[int, int]:
+def FindInTrie(trie: TRIE, s: str) -> tuple[int, TK_KIND]:
+    """ Returns (size, TK_KIND) if s[:size] is a token in the trie and 0, 0 otherwise"""
     node = trie[0]
     for n, cc in enumerate(s):
         x = node[ord(cc)]
         if x == NODE_NULL:
-            return 0, 0
+            return 0, TK_KIND.INVALID
         if x >= len(trie):
             if x >= TK_KIND_OFFSET_FOR_LOOK_AHEAD:
+                # we found a char that is not part of the kw
                 return n, TK_KIND(x - TK_KIND_OFFSET_FOR_LOOK_AHEAD)
             else:
                 return n + 1, TK_KIND(x - TK_KIND_OFFSET)
         node = trie[x]
-    return 0, 0
+    return 0, TK_KIND.INVALID
 
 
 def VerifyTrie(trie: TRIE, KWs):
@@ -259,7 +264,7 @@ def add_kw_simple(kw, tag):
     def add_kw(kw, tag, non_succ):
         # keyword is only valid if not followed by char in non_succ
         # E.g.
-        # if is a keyword but ifoo is not
+        # `if`` is a keyword but `ifoo`` is not
         # simarly
         # >> is an operator(-keyword) for most subsequent chars
         # except >>> and >>=
@@ -307,6 +312,8 @@ def add_kw(kw, tag, non_succ):
 
 def MakeTrieNoisy():
     KWs = GetAllKWAndOps()
+    for k, v in sorted(KWs):
+        print(k, v)
     trie = MakeInitialTrie(KWs)
     #
     print("Stats")
@@ -650,6 +657,13 @@ def MakePerfectHashForBinOp():
 
 if __name__ == "__main__":
     if len(sys.argv) == 1:
+        inp = Lexer(LexerRaw("stdin", sys.stdin))
+        while True:
+            tk = inp.next()
+            if tk.kind == TK_KIND.SPECIAL_EOF:
+                break
+            print(tk)
+    elif sys.argv[1] == "trie_stats":
         MakeTrieNoisy()
     elif sys.argv[1] == "gen_cc":
         cgen.ReplaceContent(GenerateCodeCC, sys.stdin, sys.stdout)

FE/parse.py

@@ -858,8 +858,8 @@ def main():
         logging.basicConfig(level=logging.WARNING)
         logger.setLevel(logging.WARNING)
         inp = lexer.Lexer(lexer.LexerRaw("stdin", sys.stdin))
-        mod = _ParseDefMod(inp)
+        mod = _ParseDefMod(inp, "dummy")
         RemoveRedundantParens(mod)
-        pp_sexpr.PrettyPrint(mod)
+        pp_sexpr.PrettyPrint(mod, sys.stdout)
 
     main()

FE/typify.py

@@ -649,6 +649,7 @@ def _TypifyExprOrType(node, tc: type_corpus.TypeCorpus,
         elif ct.is_enum():
             return _NodeSetType(node, ct.underlying_type())
         else:
+            cwast.CompilerError(node.x_srcloc, f"expected enum or wrapped type in unwrap expression but got {ct}")
             assert False
     elif isinstance(node, cwast.ExprIs):
         _TypifyExprOrType(node.type, tc, cwast.NO_TYPE, pm)