Add VIDL_STORAGE annotation for per-parameter storage type overrides

Author: hypernewbie

README.md

@@ -60,6 +60,38 @@ struct VIDLHandler
 };
 ```
 
+## Storage-type overrides
+
+By default, Vidl strips references from parameter types when generating struct
+members (so `const std::vector<int>&` becomes `std::vector`).  The
+`// VIDL_STORAGE` annotation lets you override the stored type for any
+individual parameter — useful when you want to avoid heap copies of large
+arguments.
+
+```cpp
+// VIDL_GENERATE
+// VIDL_STORAGE: uniforms = vhArenaSpan< vhArenaUniformValue >
+void vhCmdSetStateUniforms(
+    vhStateId id,
+    const std::vector< vhState::UniformBufferValue >& uniforms );
+```
+
+The generated struct stores `vhArenaSpan< … >` for `uniforms` and the ctor
+accepts that span type verbatim.  The public function signature is unchanged;
+the hand-written implementation is responsible for converting the `vector` to
+a span before constructing the VIDL struct.
+
+**Syntax rules:**
+
+- Place each `// VIDL_STORAGE: <param_name> = <storage_type>` line between
+  `// VIDL_GENERATE` and the function declaration it applies to.
+- One annotation line per parameter; multiple lines are allowed to override
+  several parameters.
+- Annotations are scoped to the immediately following function only — they
+  do not carry over to subsequent `// VIDL_GENERATE` blocks.
+- References an unknown parameter name → `ValueError`.
+- Annotates the same parameter twice → `ValueError`.
+
 ## Usage
 
 You can use `vidl.py` directly to generate code from a source file.

test.py

@@ -271,5 +271,163 @@ def test_regression_reference_members(self):
         # The constructor parameter SHOULD be a reference
         self.assertIn("VIDL_vhBeginMarker(const std::string& _name)", output)
 
+
+class TestVidlStorage(unittest.TestCase):
+
+    def test_no_annotation_unchanged(self):
+        src = """
+        // VIDL_GENERATE
+        void foo( int a, std::vector<int> b );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("int a;", out)
+        self.assertIn("std::vector<int> b;", out)
+        self.assertIn("foo(int _a, std::vector<int> _b)", out)
+
+    def test_single_storage_override(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: b = MySpan<int>
+        void foo( int a, std::vector<int> b );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("int a;", out)
+        self.assertIn("MySpan<int> b;", out)
+        self.assertNotIn("std::vector<int> b;", out)
+        self.assertIn("foo(int _a, MySpan<int> _b)", out)
+
+    def test_multiple_overrides_one_func(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: a = ArenaA
+        // VIDL_STORAGE: b = ArenaB
+        void foo( int a, int b, int c );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("ArenaA a;", out)
+        self.assertIn("ArenaB b;", out)
+        self.assertIn("int c;", out)
+        self.assertIn("foo(ArenaA _a, ArenaB _b, int _c)", out)
+
+    def test_override_with_template_args(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: data = vhArenaSpan< glm::mat4 >
+        void foo( vhStateId id, std::vector< glm::mat4 > data );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("vhArenaSpan< glm::mat4 > data;", out)
+        self.assertIn("vhStateId _id, vhArenaSpan< glm::mat4 > _data", out)
+
+    def test_whitespace_tolerance(self):
+        src = """
+        // VIDL_GENERATE
+        //VIDL_STORAGE:b=MyType
+        void foo( int a, int b );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("MyType b;", out)
+
+    def test_annotation_scoped_to_one_block(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: x = OverrideA
+        void foo( int x );
+
+        // VIDL_GENERATE
+        void bar( int x );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("OverrideA x;", out)
+        foo_idx = out.index("VIDL_foo")
+        bar_idx = out.index("VIDL_bar")
+        bar_section = out[bar_idx:]
+        self.assertNotIn("OverrideA", bar_section)
+        self.assertIn("int x;", bar_section)
+
+    def test_keyword_param_name_with_override(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: class = MyClass
+        void foo( int class );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("MyClass class_;", out)
+        self.assertIn("MyClass _class_", out)
+
+    def test_magic_constants_stable_with_override(self):
+        src_no = "// VIDL_GENERATE\nvoid foo( std::vector<int> a );"
+        src_yes = (
+            "// VIDL_GENERATE\n"
+            "// VIDL_STORAGE: a = MySpan\n"
+            "void foo( std::vector<int> a );"
+        )
+        m1 = re.search(r"kMagic = (\w+)", vidl.generate_source(src_no)).group(1)
+        m2 = re.search(r"kMagic = (\w+)", vidl.generate_source(src_yes)).group(1)
+        self.assertEqual(m1, m2)
+
+    def test_unknown_param_name_raises(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: typo = MyType
+        void foo( int a );
+        """
+        with self.assertRaises(ValueError) as ctx:
+            vidl.generate_source(src)
+        msg = str(ctx.exception)
+        self.assertIn("typo", msg)
+        self.assertIn("foo", msg)
+        self.assertIn("a", msg)
+
+    def test_duplicate_annotation_raises(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: x = TypeA
+        // VIDL_STORAGE: x = TypeB
+        void foo( int x );
+        """
+        with self.assertRaises(ValueError) as ctx:
+            vidl.generate_source(src)
+        msg = str(ctx.exception)
+        self.assertIn("duplicate", msg)
+        self.assertIn("x", msg)
+        self.assertIn("foo", msg)
+
+    def test_default_value_preserved_with_override(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: x = MyType
+        void foo( int x = 5 );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("MyType x = 5;", out)
+
+    def test_override_only_named_param_affected(self):
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: b = Overridden
+        void foo( std::vector<int> a, std::vector<int> b, std::vector<float> c );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("std::vector<int> a;", out)
+        self.assertIn("Overridden b;", out)
+        self.assertIn("std::vector<float> c;", out)
+        self.assertNotIn("std::vector<int> b;", out)
+
+    def test_ctor_param_uses_override_not_reference(self):
+        # Without override, a reference param like `const std::vector<int>& v`
+        # strips the & for the member type but keeps it in the ctor.
+        # With an override the ctor must use the override type verbatim.
+        src = """
+        // VIDL_GENERATE
+        // VIDL_STORAGE: v = MySpan
+        void foo( const std::vector<int>& v );
+        """
+        out = vidl.generate_source(src)
+        self.assertIn("MySpan v;", out)
+        self.assertIn("MySpan _v", out)
+        self.assertNotIn("MySpan& _v", out)
+
+
 if __name__ == '__main__':
     unittest.main()

vidl.py

@@ -21,6 +21,28 @@
 
 # WARNING: Vibe code generation tool ahead. Read at your own risk of sanity.
 
+# ----------------------------------------------------------------------------
+# // VIDL_STORAGE — per-parameter storage type override
+# ----------------------------------------------------------------------------
+#
+# Form:
+#
+#     // VIDL_GENERATE
+#     // VIDL_STORAGE: <param_name> = <storage_type>
+#     ReturnType funcname( ... );
+#
+# Replaces the generated struct member type and matching ctor argument for
+# <param_name> with <storage_type>, verbatim. The function declaration itself
+# is left alone — converting the original argument into the storage type is
+# the caller's job. Motivating case: letting a setter that takes
+# `const std::vector<X>&` store a non-owning arena span, so enqueueing the
+# command does not heap-copy the vector.
+#
+# Multiple lines per block are allowed (one per overridden param). Scope is
+# strictly the next `// VIDL_GENERATE` block. Naming an unknown parameter, or
+# annotating the same parameter twice, raises `ValueError`.
+# ----------------------------------------------------------------------------
+
 import os, sys, re, hashlib
 
 def generate_magic(name):
@@ -96,48 +118,95 @@ def parse_function(text):
     }
 
 def generate_source(content):
-    # Find all // VIDL_GENERATE
-    # Then find the function signature that follows
-    # We look for // VIDL_GENERATE and then everything until a semicolon
+    """Parse *content* for // VIDL_GENERATE blocks and return a string
+    of generated C++ structs plus a VIDLHandler dispatcher.
+
+    Supports per-parameter storage-type overrides via // VIDL_STORAGE
+    annotations placed between // VIDL_GENERATE and the function declaration.
+    See the top-of-file comment block in vidl.py for the full syntax.
+
+    Raises ``ValueError`` when a VIDL_STORAGE annotation references an
+    unknown parameter name or when the same parameter is annotated twice
+    within a single block.
+    """
     pattern = r'//\s*VIDL_GENERATE(.*?);'
     matches = re.findall(pattern, content, re.DOTALL)
 
     generated_structs = []
     handler_funcs = []
 
     for m in matches:
+        # Collect VIDL_STORAGE annotations into a list *before* stripping
+        # comments so that we can validate duplicates and unknown params
+        # once we know the function's parameter names.
+        storage_annotations = []
+        for sm in re.finditer(
+                r'//\s*VIDL_STORAGE\s*:\s*(\w+)\s*=\s*(.+)', m):
+            storage_annotations.append(
+                (sm.group(1).strip(), sm.group(2).strip()))
+
         # Strip comments from the captured block
         m = re.sub(r'//.*', '', m)
         m = re.sub(r'/\*.*?\*/', '', m, flags=re.DOTALL)
-        
+
         func = parse_function(m)
         if not func:
             continue
-        
+
+        # Validate annotations and build storage_overrides dict
+        valid_names = {p['orig_name'] for p in func['params']}
+        valid_names |= {p['name'] for p in func['params']}
+
+        storage_overrides = {}
+        seen_params = set()
+        for param_key, storage_type in storage_annotations:
+            if param_key in seen_params:
+                raise ValueError(
+                    f"VIDL_STORAGE: duplicate annotation for parameter "
+                    f"'{param_key}' in function '{func['name']}'"
+                )
+            seen_params.add(param_key)
+            if param_key not in valid_names:
+                orig_names = ', '.join(
+                    p['orig_name'] for p in func['params'])
+                raise ValueError(
+                    f"VIDL_STORAGE: parameter '{param_key}' not found "
+                    f"in function '{func['name']}' "
+                    f"(params: {orig_names})"
+                )
+            storage_overrides[param_key] = storage_type
+
         magic = generate_magic(func['name'])
         handler_funcs.append({'name': func['name'], 'magic': magic})
-        
+
         # Generate struct
         struct_lines = [f"struct VIDL_{func['name']}", "{"]
         struct_lines.append(f"    static constexpr uint64_t kMagic = {magic};")
         struct_lines.append("    uint64_t MAGIC = kMagic;")
-        
+
         ctor_params = []
         initializer_list = []
-        
+
         for p in func['params']:
-            member_type = p['type'].replace('&', '').strip()
+            override = storage_overrides.get(p['orig_name']) or storage_overrides.get(p['name'])
+            if override is not None:
+                # Use the override storage type for both the struct member and the ctor param.
+                member_type = override
+                ctor_param_type = override
+            else:
+                member_type = p['type'].replace('&', '').strip()
+                ctor_param_type = p['type']
             default_str = f" = {p['default']}" if p.get('default') else ""
             struct_lines.append(f"    {member_type} {p['name']}{default_str};")
             # For constructor: type _name
-            ctor_params.append(f"{p['type']} _{p['name']}")
+            ctor_params.append(f"{ctor_param_type} _{p['name']}")
             # For initializer: name(_name)
             initializer_list.append(f"{p['name']}(_{p['name']})")
-        
+
         # Default constructor
         struct_lines.append("")
         struct_lines.append(f"    VIDL_{func['name']}() = default;")
-        
+
         # Parameterized constructor
         if ctor_params:
             struct_lines.append("")
@@ -191,7 +260,11 @@ def main():
     with open(input_file, 'r', encoding='utf-8') as f:
         content = f.read()
 
-    result = generate_source(content)
+    try:
+        result = generate_source(content)
+    except ValueError as e:
+        print(f"vidl.py error: {e}", file=sys.stderr)
+        sys.exit(1)
 
     if len(sys.argv) >= 3:
         output_file = sys.argv[2]