Merge pull request #57 from pingelit/feature/py-schema-dev

Real life/proof of concept development for python and JSON schema

Author: pingelit

code-gen/generate_data_structures.cmake

@@ -323,13 +323,25 @@ function (generate_data_structures TARGET_LIBRARY)
 				)
 			endif ()
 
+			if (GEN_DATA_OUTPUT_PYTHON_PKG)
+				if (NOT EXISTS ${GEN_DATA_IN_SOURCE_PATH}/${GEN_DATA_OUTPUT_PYTHON_PKG})
+					file (MAKE_DIRECTORY ${GEN_DATA_IN_SOURCE_PATH})
+				endif ()
+
+				file (COPY ${GEN_DATA_OUTPUT_BASE_DIR}/${GEN_DATA_OUTPUT_PYTHON_PKG}
+					  DESTINATION ${GEN_DATA_IN_SOURCE_PATH}/${GEN_DATA_OUTPUT_PYTHON_PKG}
+				)
+			endif ()
+
 			if (GEN_DATA_OUTPUT_SCHEMA)
 				if (NOT EXISTS ${GEN_DATA_IN_SOURCE_PATH}/${GEN_DATA_OUTPUT_SCHEMA})
 					file (MAKE_DIRECTORY ${GEN_DATA_IN_SOURCE_PATH})
 				endif ()
 
+				get_filename_component (GEN_DATA_OUTPUT_SCHEMA_PARENT_DIR ${GEN_DATA_IN_SOURCE_PATH}/${GEN_DATA_OUTPUT_SCHEMA} DIRECTORY)
+
 				file (COPY ${GEN_DATA_OUTPUT_BASE_DIR}/${GEN_DATA_OUTPUT_SCHEMA}
-					  DESTINATION ${GEN_DATA_IN_SOURCE_PATH}/${GEN_DATA_OUTPUT_SCHEMA}
+					  DESTINATION ${GEN_DATA_OUTPUT_SCHEMA_PARENT_DIR}
 				)
 			endif ()
 		endif ()

code-gen/src/poly_scribe_code_gen/py_gen.py

@@ -144,25 +144,39 @@ def _render_pyproject_toml(additional_data: AdditionalData) -> str:
 
 
 def _transform_types(parsed_idl: ParsedIDL) -> ParsedIDL:
+    defined_types = set()
+
+    for struct_name in parsed_idl["structs"]:
+        defined_types.add(struct_name)
+
+    for enum_name in parsed_idl["enums"]:
+        defined_types.add(enum_name)
+
+    for typedef_name in parsed_idl["typedefs"]:
+        defined_types.add(typedef_name)
+
     for struct_name, struct_data in parsed_idl["structs"].items():
         for member_data in struct_data["members"].values():
-            member_data["type"] = _transformer(member_data["type"], parsed_idl["inheritance_data"])
+            member_data["type"] = _transformer(member_data["type"], parsed_idl["inheritance_data"], defined_types)
 
             if member_data["default"] == "{}" and member_data["default_type"] is not None:
                 # member_data["default"] = f"Field(default={member_data['default_type']}())"
                 member_data["default"] = f"{member_data['default_type']}()"
 
             if member_data["default"] == "{}" and member_data["default_type"] is None:
                 # member_data["default"] = f"Field(default={member_data['type']}())"
-                member_data["default"] = f"{member_data['type']}()"
+                type_str = member_data["type"]
+                if isinstance(type_str, str) and type_str.startswith('"') and type_str.endswith('"'):
+                    type_str = type_str[1:-1]
+                member_data["default"] = f"{type_str}()"
 
             if not member_data["required"] and member_data["default"] is None:
                 member_data["type"] = f"Optional[{member_data['type']}]"
 
             if "str" in member_data["type"] and member_data["default"] is not None:
                 member_data["default"] = f'"{member_data["default"]}"'
 
-            if member_data["default"] is None:
+            if member_data["default"] is None and member_data["required"] is False:
                 member_data["default"] = "None"
 
         # Check if a member named "type" is already present in the struct and raise an error if so
@@ -186,14 +200,14 @@ def _transform_types(parsed_idl: ParsedIDL) -> ParsedIDL:
             }
 
     for type_def in parsed_idl["typedefs"].values():
-        type_def["type"] = _transformer(type_def["type"], parsed_idl["inheritance_data"])
+        type_def["type"] = _transformer(type_def["type"], parsed_idl["inheritance_data"], defined_types)
 
     return parsed_idl
 
 
-def _transformer(type_input: dict[str, Any], inheritance_data: dict[str, list[str]]) -> str:
+def _transformer(type_input: dict[str, Any], inheritance_data: dict[str, list[str]], defined_types: set[str]) -> str:
     if isinstance(type_input, str):
-        type_input = _get_polymorphic_type(type_input, inheritance_data)
+        type_input_poly = _get_polymorphic_type(type_input, inheritance_data, defined_types)
 
         conversion = {
             "string": "str",
@@ -214,14 +228,20 @@ def _transformer(type_input: dict[str, Any], inheritance_data: dict[str, list[st
             "unsigned long long": "int",
         }
 
-        return conversion.get(type_input, type_input)
+        # check if type_input is in defined_types
+        if type_input_poly in defined_types:
+            return f'"{conversion.get(type_input_poly, type_input_poly)}"'
+
+        return conversion.get(type_input_poly, type_input_poly)
 
     if type_input["union"]:
-        contained_types = [_transformer(contained, inheritance_data) for contained in type_input["type_name"]]
+        contained_types = [
+            _transformer(contained, inheritance_data, defined_types) for contained in type_input["type_name"]
+        ]
         transformed_type = ",".join(contained_types)
         return f"Union[{transformed_type}]"
     if type_input["vector"]:
-        transformed_type = _transformer(type_input["type_name"], inheritance_data)
+        transformed_type = _transformer(type_input["type_name"], inheritance_data, defined_types)
 
         if type_input["size"] is not None:
             return (
@@ -232,17 +252,17 @@ def _transformer(type_input: dict[str, Any], inheritance_data: dict[str, list[st
 
         return f"List[{transformed_type}]"
     if type_input["map"]:
-        key_type = _transformer(type_input["type_name"]["key"], inheritance_data)
+        key_type = _transformer(type_input["type_name"]["key"], inheritance_data, defined_types)
         # TODO: here we can check if the type changed??
 
-        value_type = _transformer(type_input["type_name"]["value"], inheritance_data)
+        value_type = _transformer(type_input["type_name"]["value"], inheritance_data, defined_types)
         return f"Dict[{key_type}, {value_type}]"
 
     msg = f"Unknown type: {type_input}"
     raise ValueError(msg)
 
 
-def _get_polymorphic_type(type_input: str, inheritance_data: dict[str, list[str]]) -> str:
+def _get_polymorphic_type(type_input: str, inheritance_data: dict[str, list[str]], defined_types: set[str]) -> str:
     union_content = []
 
     if type_input in inheritance_data:
@@ -255,6 +275,7 @@ def _get_polymorphic_type(type_input: str, inheritance_data: dict[str, list[str]
             union_content.append(base_type)
 
     if union_content:
+        union_content = [f'"{type_name}"' for type_name in union_content if type_name in defined_types]
         return f"Annotated[Union[{', '.join(union_content)}],Field(discriminator=\"type\")]"
 
     return type_input

code-gen/src/poly_scribe_code_gen/templates/python.jinja

@@ -61,6 +61,10 @@ class {{ struct_name }}{% if struct_data["inheritance"] %}({{ struct_data["inher
 
 {% endfor%}
 
+{% for struct_name in structs %}
+{{ struct_name }}.model_rebuild()
+{% endfor %}
+
 def load(model_type: Type[T], file: Union[Path, str]) -> T:
     if isinstance(file, str):
         file = Path(file).resolve()

code-gen/tests/py_gen_test.py

@@ -101,7 +101,7 @@ def test__transform_types_structs() -> None:
     )
     assert (
         result["structs"]["BazQux"]["members"]["union"]["type"].replace(" ", "")
-        == "Optional[Union[int,float,bool,FooBar]]"
+        == 'Optional[Union[int,float,bool,"FooBar"]]'
     )
 
 
@@ -115,7 +115,7 @@ def test__transform_types_unknown_type() -> None:
     }
 
     with pytest.raises(ValueError, match="Unknown type:"):
-        py_gen._transformer(type_data, {})
+        py_gen._transformer(type_data, {}, set())
 
 
 def test_render_template_typedefs() -> None:
@@ -191,7 +191,7 @@ def test_render_template_structs() -> None:
                 " ", ""
             ) in struct_body.replace(" ", "")
         elif match[0] == "BazQux":
-            assert "union: Optional[Union[int, float, bool, FooBar]]".replace(" ", "") in struct_body.replace(" ", "")
+            assert 'union: Optional[Union[int, float, bool, "FooBar"]]'.replace(" ", "") in struct_body.replace(" ", "")
 
 
 def test_render_template_struct_with_inheritance() -> None:
@@ -269,7 +269,7 @@ def test_render_template_struct_with_poly_inheritance() -> None:
             assert "baz: float".replace(" ", "") in struct_body.replace(" ", "")
             assert 'type: Literal["C"] = "C"'.replace(" ", "") in struct_body.replace(" ", "")
         elif match[0] == "Y":
-            assert 'content: Annotated[Union[B,C,X],Field(discriminator="type")]'.replace(
+            assert 'content: Annotated[Union["B","C","X"],Field(discriminator="type")]'.replace(
                 " ", ""
             ) in struct_body.replace(" ", "")
 
@@ -400,7 +400,7 @@ def test__render_template_poly_derived_in_decl() -> None:
     for match in matches:
         struct_body = match[2]
         if match[0] == "Y":
-            assert 'content: Optional[Annotated[Union[B, C, X], Field(discriminator="type")]] = None'.replace(
+            assert 'content: Optional[Annotated[Union["B", "C", "X"], Field(discriminator="type")]] = None'.replace(
                 " ", ""
             ) in struct_body.replace(" ", "")
 
@@ -598,7 +598,7 @@ def test_render_template_struct_with_empty_type_default() -> None:
     for match in matches:
         struct_body = match[2]
         if match[0] == "Data":
-            assert 'base: Annotated[Union[Foo, Bar, Base],Field(discriminator="type")] = Foo()'.replace(
+            assert 'base: Annotated[Union["Foo", "Bar", "Base"],Field(discriminator="type")] = Foo()'.replace(
                 " ", ""
             ) in struct_body.replace(" ", "")
         elif match[0] == "Base":