Phase 3

Signed-off-by: Christian Vetter <christian.vetter@here.com>

Author: VeaaC

flatdata-generator/flatdata/generator/app.py

@@ -21,7 +21,7 @@
 from flatdata.generator.tree.errors import FlatdataSyntaxError
 
 
-def _parse_command_line():
+def _parse_command_line() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Generates code for a given flatdata schema file.")
     parser.add_argument("-s", "--schema", type=str, required=True,
@@ -39,7 +39,7 @@ def _parse_command_line():
     return parser.parse_args()
 
 
-def _setup_logging(args):
+def _setup_logging(args: argparse.Namespace) -> None:
     level = logging.WARNING
     if args.debug:
         level = logging.DEBUG
@@ -52,13 +52,13 @@ def _setup_logging(args):
         level=level)
 
 
-def _check_args(args):
+def _check_args(args: argparse.Namespace) -> None:
     if not os.path.isfile(args.schema):
         logging.fatal("Cannot find schema file at %s", args.schema)
         sys.exit(1)
 
 
-def _run(args):
+def _run(args: argparse.Namespace) -> None:
     _setup_logging(args)
     _check_args(args)
 
@@ -86,6 +86,6 @@ def _run(args):
         logging.info("Code for %s is written to %s", args.gen, args.output_file)
 
 
-def main():
+def main() -> None:
     """Entrypoint"""
     _run(_parse_command_line())

flatdata-generator/flatdata/generator/engine.py

@@ -4,17 +4,20 @@
 '''
 
 import types
+from typing import Any
 
 from flatdata.generator.tree.builder import build_ast
 from flatdata.generator.tree.nodes.trivial.namespace import Namespace
 from flatdata.generator.tree.nodes.node import Node
+from flatdata.generator.tree.syntax_tree import SyntaxTree
 
 from .generators.cpp import CppGenerator
 from .generators.dot import DotGenerator
 from .generators.go import GoGenerator
 from .generators.python import PythonGenerator
 from .generators.rust import RustGenerator
 from .generators.flatdata import FlatdataGenerator
+from .generators import BaseGenerator
 
 
 class Engine:
@@ -23,7 +26,7 @@ class Engine:
     Implements code generation from the given flatdata schema.
     """
 
-    _GENERATORS = {
+    _GENERATORS: dict[str, type[BaseGenerator]] = {
         "cpp": CppGenerator,
         "dot": DotGenerator,
         "go": GoGenerator,
@@ -33,21 +36,21 @@ class Engine:
     }
 
     @classmethod
-    def available_generators(cls):
+    def available_generators(cls) -> list[str]:
         """
         Lists names of available code generators.
         """
         return list(cls._GENERATORS.keys())
 
-    def __init__(self, schema):
+    def __init__(self, schema: str) -> None:
         """
         Instantiates generator engine for a given schema.
         :raises FlatdataSyntaxError
         """
         self.schema = schema
         self.tree = build_ast(schema)
 
-    def render(self, generator_name):
+    def render(self, generator_name: str) -> str:
         """
         Render schema with a given generator
         :param generator_name:
@@ -60,38 +63,38 @@ def render(self, generator_name):
             )
 
         output_content = generator.render(self.tree)
-        return output_content
+        return str(output_content)
 
-    def render_python_module(self, module_name=None, archive_name=None, root_namespace=None):
+    def render_python_module(self, module_name: str | None = None, archive_name: str | None = None, root_namespace: str | None = None) -> types.ModuleType | tuple[types.ModuleType, Any]:
         """
         Render python module.
         :param module_name: Module name to use. If none, root namespace name is used.
         :param archive_name: Archive name to lookup,
             if specified, archive type is returned along with the model
         :param root_namespace: Root namespace to pick in case of multiple top level namespaces.
         """
-        root_namespace = self._find_root_namespace(self.tree, archive_name, root_namespace)
+        ns = self._find_root_namespace(self.tree, archive_name, root_namespace)
         module_code = self.render("py")
-        module = types.ModuleType(module_name if module_name is not None else root_namespace.name)
+        module = types.ModuleType(module_name if module_name is not None else ns.name)
         #pylint: disable=exec-used
         exec(module_code, module.__dict__)
         if archive_name is None:
             return module
 
-        name = root_namespace.name + "_" + archive_name
+        name = ns.name + "_" + archive_name
         archive_type = getattr(module, name) if archive_name else None
         return module, archive_type
 
     @classmethod
-    def _create_generator(cls, name):
+    def _create_generator(cls, name: str) -> BaseGenerator | None:
         generator_type = cls._GENERATORS.get(name, None)
         if generator_type is None:
             return None
 
-        return generator_type()  # type: ignore[abstract]  # dict values are concrete subclasses
+        return generator_type()  # type: ignore[call-arg]  # dict values are concrete subclasses with zero-arg __init__
 
     @staticmethod
-    def _find_root_namespace(tree, archive_name, root_namespace=None):
+    def _find_root_namespace(tree: SyntaxTree, archive_name: str | None, root_namespace: str | None = None) -> Namespace:
         root_children = tree.root.children
         root_namespaces = [
             child for child in root_children

flatdata-generator/flatdata/generator/generators/__init__.py

@@ -4,6 +4,8 @@
 '''
 
 from abc import ABCMeta, abstractmethod
+from typing import Any
+
 from jinja2 import Environment, PackageLoader
 from jinja2 import nodes
 from jinja2.ext import Extension
@@ -21,21 +23,21 @@
 class BaseGenerator(metaclass=ABCMeta):
     """Abstract base class for Flatdata generators"""
 
-    def __init__(self, template):
+    def __init__(self, template: str) -> None:
         self._template = template
 
     @abstractmethod
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         """List of supported nodes by this generator"""
         raise RuntimeError(
             "Derived generators must implement _supported_nodes")
 
     @abstractmethod
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         raise RuntimeError(
             "Derived generators must implement _populate_filters")
 
-    def render(self, tree):
+    def render(self, tree: Any) -> str:
         """Generate the language implementation from the AST"""
         env = Environment(loader=PackageLoader('flatdata.generator', 'templates'), lstrip_blocks=True,
                           trim_blocks=True, autoescape=False, extensions=[RaiseExtension])
@@ -71,7 +73,7 @@ class RaiseExtension(Extension):
 
     tags = set(['raise'])
 
-    def parse(self, parser):
+    def parse(self, parser: Any) -> nodes.CallBlock:
         """The first token is the line number, followed by the expression"""
         lineno = next(parser.stream).lineno
         message_node = parser.parse_expression()
@@ -81,6 +83,6 @@ def parse(self, parser):
         )
 
     #pylint: disable=no-self-use
-    def _raise(self, msg, caller):
+    def _raise(self, msg: str, caller: Any) -> str:
         """Helper callback."""
         raise TemplateRuntimeError(msg)

flatdata-generator/flatdata/generator/generators/cpp.py

@@ -3,6 +3,8 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
+from jinja2 import Environment
+
 from flatdata.generator.tree.nodes.resources import Vector, Multivector, Instance, RawData, BoundResource, \
     ResourceBase, Archive as ArchiveResource
 from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant, Field
@@ -13,13 +15,13 @@
 class CppGenerator(BaseGenerator):
     """Flatdata to C++ header file generator"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "cpp/cpp.jinja2")
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Structure, Archive, Constant, Enumeration]
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         env.filters["cpp_doc"] = lambda value: value
 
         def _safe_cpp_string_line(value):

flatdata-generator/flatdata/generator/generators/dot.py

@@ -6,17 +6,19 @@
 from flatdata.generator.tree.nodes.archive import Archive
 from . import BaseGenerator
 
+from jinja2 import Environment
+
 SCOPE_SEPARATOR = "__"
 DECORATION_BOUND = "__bound__"
 
 
 class DotGenerator(BaseGenerator):
     """Flatdata to DOT (graph description language) generator"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "dot/dot.jinja2")
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         env.autoescape = True
 
         def _field_value_type(field):
@@ -31,5 +33,5 @@ def _field_value_type(field):
 
         env.filters["field_value_type"] = _field_value_type
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Archive]

flatdata-generator/flatdata/generator/generators/flatdata.py

@@ -3,6 +3,8 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
+from jinja2 import Environment
+
 from flatdata.generator.tree.nodes.resources import BoundResource
 from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant
 from flatdata.generator.tree.nodes.archive import Archive
@@ -13,13 +15,13 @@
 class FlatdataGenerator(BaseGenerator):
     """Flatdata to Flatdata generator, used for debugging/testing"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "flatdata/flatdata.jinja2")
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Structure, Archive, Constant, Enumeration]
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         def _is_builtin(node):
             for namespace in SyntaxTree.namespaces(node):
                 if namespace.name == "_builtin":

flatdata-generator/flatdata/generator/generators/go.py

@@ -2,6 +2,8 @@
  Copyright (c) 2017 HERE Europe B.V.
  See the LICENSE file in the root of this project for license details.
 '''
+from jinja2 import Environment
+
 from flatdata.generator.tree.nodes.archive import Archive
 from flatdata.generator.tree.nodes.node import Node
 from flatdata.generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
@@ -14,13 +16,13 @@
 class GoGenerator(BaseGenerator):
     """Flatdata to Go generator"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "go/go.jinja2")
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Structure, Archive, Constant]
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         def _decorate_archive_type(value):
             assert isinstance(value, Node)
             return value.name
@@ -70,7 +72,7 @@ def to_initializer(resource, tree):
                      for t in resource.referenced_structures]
                 ))
             if isinstance(resource, ArchiveResource):
-                return _decorate_archive_type(resource.children[0].node)
+                return _decorate_archive_type(resource.children[0].node)  # type: ignore[attr-defined]  # child is an ArchiveReference which has .node
             if isinstance(resource, RawData):
                 return "None"
             raise ValueError("Unknown resource type: %s" % (resource.__class__))

flatdata-generator/flatdata/generator/generators/python.py

@@ -2,6 +2,8 @@
  Copyright (c) 2017 HERE Europe B.V.
  See the LICENSE file in the root of this project for license details.
 '''
+from jinja2 import Environment
+
 from flatdata.generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
 from flatdata.generator.tree.nodes.resources.archive import Archive as ArchiveResource
 from flatdata.generator.tree.nodes.trivial import Structure
@@ -13,13 +15,13 @@
 class PythonGenerator(BaseGenerator):
     """Flatdata to Python generator"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "py/python.jinja2")
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Structure, Archive]
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         def _decorate_archive_type(tree, value):
             assert isinstance(value, Node)
             return tree.namespace_path(value, "_") + "_" + value.name
@@ -54,7 +56,7 @@ def to_initializer(resource, tree):
                     ','.join([_decorate_archive_type(tree, t.node) for t in
                               resource.referenced_structures]))
             if isinstance(resource, ArchiveResource):
-                return _decorate_archive_type(tree, resource.children[0].node)
+                return _decorate_archive_type(tree, resource.children[0].node)  # type: ignore[attr-defined]  # child is an ArchiveReference which has .node
             if isinstance(resource, RawData):
                 return "None"
             raise ValueError("Unknown resource type: %s" % (resource.__class__))

flatdata-generator/flatdata/generator/generators/rust.py

@@ -4,6 +4,8 @@
 '''
 import re
 
+from jinja2 import Environment
+
 from flatdata.generator.tree.nodes.resources import (Vector, Multivector, Instance, RawData, BoundResource,
                                             Archive as ArchiveResource)
 from flatdata.generator.tree.nodes.trivial import Structure, Constant, Enumeration
@@ -23,14 +25,14 @@ class RustGenerator(BaseGenerator):
         "pure", "ref", "return", "self", "sizeof", "static", "struct", "super", "trait", "true",
         "type", "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield"]
 
-    def __init__(self):
+    def __init__(self) -> None:
         BaseGenerator.__init__(self, "rust/rust.jinja2")
 
-    def supported_nodes(self):
+    def supported_nodes(self) -> list[type]:
         return [Structure, Archive, Constant, Enumeration]
 
     @staticmethod
-    def _format_numeric_literal(value):
+    def _format_numeric_literal(value: str) -> str:
         try:
             # only apply this to integer values
             number = int(value)
@@ -40,7 +42,7 @@ def _format_numeric_literal(value):
         except ValueError:
             return value
 
-    def _populate_environment(self, env):
+    def _populate_environment(self, env: Environment) -> None:
         def _camel_to_snake_case(expr):
             step1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', expr)
             return re.sub('([a-z0-9])(A-Z)', r'\1_\2', step1).lower()