fix(codegen): treat dict-typed fields as leaf values in examples

flatten_example recursed into all dicts, splitting dict-typed fields
like `tags: dict[str, str]` into dot-notation rows. Now
collect_dict_paths walks the FieldSpec tree to identify dict-typed
field paths, and _flatten_value checks membership before recursing.

Indexed runtime paths (items[0].tags) are normalized to schema
notation (items[].tags) for matching. The pipeline computes
dict_paths from spec.fields and threads them through load_examples.

Also: clarify mutual exclusion in type visitor elif chains
(reverse_references, type_collection) and rename _TypeIdentity to
_TypeShape in union_extraction to avoid shadowing specs.TypeIdentity.

Author: sethfitz

packages/overture-schema-codegen/docs/design.md

@@ -240,6 +240,12 @@ Loads example data from theme `pyproject.toml` files, validates against Pydantic
 and flattens to dot-notation rows for display in feature pages. Also provides a starting
 point for generated test data.
 
+`collect_dict_paths` walks the `FieldSpec` tree to identify dict-typed fields (like
+`tags: dict[str, str]`), returning their dot-paths as a `frozenset`. `flatten_example`
+checks this set before recursing into dicts -- paths in the set are kept as leaf values
+rather than being split into dot-notation rows. The pipeline computes `dict_paths` from
+`spec.fields` and threads it through `load_examples`.
+
 ## Extension Points
 
 **Adding a new output target** (Arrow schemas next, PySpark expressions after): Add a

packages/overture-schema-codegen/docs/walkthrough.md

@@ -609,8 +609,15 @@ non-selected variant arms. `_strip_null_unknown_fields` removes null-valued fiel
 in the common base's field set, so the selected arm's validator accepts the data without
 choking on fields that belong to sibling variants.
 
+`collect_dict_paths` walks the `FieldSpec` tree to identify dict-typed fields (like
+`tags: dict[str, str]`), returning their dot-paths as a `frozenset`. Schema-notation
+paths use empty brackets (`items[].tags`) while runtime paths carry indices
+(`items[0].tags`); `_normalize_path` strips indices before membership checks.
+
 `flatten_example` converts nested dicts to dot-notation. Nested dicts become
-`parent.child`, lists of dicts become `parent[0].child`. `order_example_rows` sorts by
+`parent.child`, lists of dicts become `parent[0].child`. Dicts at paths in `dict_paths`
+are kept as leaf values -- a `tags` field typed as `dict[str, str]` renders as a whole
+map rather than being split into `tags.color`, `tags.size`. `order_example_rows` sorts by
 field position in the documentation's field order using a stable sort, so sub-fields
 maintain their original relative order.
 
@@ -732,8 +739,9 @@ sources appear on the source NewType's page instead.
 
 The example loader finds `pyproject.toml` in the transportation theme package, reads
 `[examples.Segment]`, validates each example against the union alias (injecting literal
-fields, stripping null fields from non-selected arms), flattens to dot-notation, and
-orders by field position.
+fields, stripping null fields from non-selected arms), computes `dict_paths` from
+`spec.fields` to identify dict-typed fields, flattens to dot-notation (keeping dict-typed
+fields as leaf values), and orders by field position.
 
 The Jinja2 template assembles the field table, optional constraints section, examples,
 and "Used By" partial into markdown.

packages/overture-schema-codegen/src/overture/schema/codegen/example_loader.py

@@ -1,6 +1,7 @@
 """Load and process example data from theme pyproject.toml files."""
 
 import logging
+import re
 import sys
 from dataclasses import dataclass
 from pathlib import Path
@@ -10,11 +11,12 @@
 from pydantic.fields import FieldInfo
 
 from .model_extraction import resolve_field_alias
+from .specs import FieldSpec
 from .type_analyzer import single_literal_value
 
 log = logging.getLogger(__name__)
 
-__all__ = ["ExampleRecord", "load_examples", "validate_example"]
+__all__ = ["ExampleRecord", "collect_dict_paths", "load_examples", "validate_example"]
 
 # tomllib is stdlib from 3.11+; tomli is the backport for 3.10.
 try:
@@ -140,19 +142,63 @@ def validate_example(
 
 
 _DEFAULT_SKIP_KEYS: frozenset[str] = frozenset()
+_DEFAULT_DICT_PATHS: frozenset[str] = frozenset()
 
+_INDEXED_BRACKET = re.compile(r"\[\d+\]")
 
-def _flatten_value(prefix: str, value: object) -> list[tuple[str, Any]]:
+
+def _normalize_path(path: str) -> str:
+    """Replace indexed brackets with empty brackets for dict_paths matching.
+
+    ``collect_dict_paths`` produces schema-notation paths like
+    ``items[].tags``, while ``_flatten_value`` builds runtime paths like
+    ``items[0].tags``. Normalizing before membership testing makes them
+    comparable.
+    """
+    return _INDEXED_BRACKET.sub("[]", path)
+
+
+def collect_dict_paths(fields: list[FieldSpec], prefix: str = "") -> frozenset[str]:
+    """Collect dot-paths of dict-typed fields from a FieldSpec tree.
+
+    Walks the ``FieldSpec.model`` tree (same structure the renderer walks
+    for inline expansion) and returns paths where ``type_info.is_dict``
+    is True. These paths tell ``flatten_example`` which dicts are maps
+    (keep as leaf) vs. models (recurse into).
+
+    Parameters
+    ----------
+    fields : list[FieldSpec]
+        Fields to walk.
+    prefix : str
+        Dot-notation prefix accumulated from parent fields.
+    """
+    paths: set[str] = set()
+    for f in fields:
+        path = f"{prefix}{f.name}" if prefix else f.name
+        if f.type_info.is_dict:
+            paths.add(path)
+        elif f.model and not f.starts_cycle:
+            suffix = "[]" * f.type_info.list_depth if f.type_info.is_list else ""
+            paths |= collect_dict_paths(f.model.fields, f"{path}{suffix}.")
+    return frozenset(paths)
+
+
+def _flatten_value(
+    prefix: str, value: object, dict_paths: frozenset[str]
+) -> list[tuple[str, Any]]:
     """Recursively flatten a value into dot/bracket-notation rows."""
     if isinstance(value, dict):
+        if _normalize_path(prefix) in dict_paths:
+            return [(prefix, value)]
         result: list[tuple[str, Any]] = []
         for k, v in value.items():
-            result.extend(_flatten_value(f"{prefix}.{k}", v))
+            result.extend(_flatten_value(f"{prefix}.{k}", v, dict_paths))
         return result
     if isinstance(value, list) and value and isinstance(value[0], (dict, list)):
         result = []
         for i, item in enumerate(value):
-            result.extend(_flatten_value(f"{prefix}[{i}]", item))
+            result.extend(_flatten_value(f"{prefix}[{i}]", item, dict_paths))
         return result
     return [(prefix, value)]
 
@@ -161,19 +207,24 @@ def flatten_example(
     raw: dict[str, Any],
     *,
     skip_keys: frozenset[str] = _DEFAULT_SKIP_KEYS,
+    dict_paths: frozenset[str] = _DEFAULT_DICT_PATHS,
 ) -> list[tuple[str, Any]]:
     """Flatten nested example dict to dot-notation key-value pairs.
 
     Nested dicts become ``"parent.child"``; lists of dicts become
     ``"parent[0].child"``; lists of lists of dicts use double-index
     notation ``"parent[0][1].child"``. Keys in *skip_keys* are dropped
     at the top level only. Plain lists are kept as values.
+
+    Dicts at paths in *dict_paths* are kept as leaf values instead of
+    being recursed into. Use ``collect_dict_paths`` to compute this set
+    from a FieldSpec tree.
     """
     result: list[tuple[str, Any]] = []
     for key, value in raw.items():
         if key in skip_keys:
             continue
-        result.extend(_flatten_value(key, value))
+        result.extend(_flatten_value(key, value, dict_paths))
     return result
 
 
@@ -257,6 +308,7 @@ def load_examples(
     *,
     pyproject_source: type | None = None,
     model_fields: dict[str, FieldInfo] | None = None,
+    dict_paths: frozenset[str] = _DEFAULT_DICT_PATHS,
 ) -> list[ExampleRecord]:
     """Load examples for a model, flattened and ordered by *field_names*.
 
@@ -278,6 +330,9 @@ def load_examples(
     model_fields : dict[str, FieldInfo] or None
         Field info dict for Literal injection. If None, infers
         from validation_type if it's a BaseModel class.
+    dict_paths : frozenset[str]
+        Dot-paths of dict-typed fields to keep as leaf values.
+        Use ``collect_dict_paths`` to compute from a FieldSpec tree.
     """
     source_type = pyproject_source if pyproject_source is not None else validation_type
     if not isinstance(source_type, type):
@@ -308,7 +363,7 @@ def load_examples(
                 e,
             )
             continue
-        flat_rows = flatten_example(denulled)
+        flat_rows = flatten_example(denulled, dict_paths=dict_paths)
         ordered_rows = order_example_rows(flat_rows, field_names)
         records.append(ExampleRecord(rows=ordered_rows))
 

packages/overture-schema-codegen/src/overture/schema/codegen/markdown_pipeline.py

@@ -13,7 +13,7 @@
 import overture.schema.system.primitive as _system_primitive
 from overture.schema.system.primitive import GeometryType
 
-from .example_loader import ExampleRecord, load_examples
+from .example_loader import ExampleRecord, collect_dict_paths, load_examples
 from .link_computation import LinkContext
 from .markdown_renderer import (
     render_enum,
@@ -74,12 +74,14 @@ def _load_model_examples(
     if not pyproject_source:
         return None
     field_names = [f.name for f in spec.fields]
+    dict_paths = collect_dict_paths(spec.fields)
     examples = load_examples(
         validation_type,
         spec.name,
         field_names,
         pyproject_source=pyproject_source,
         model_fields=model_fields,
+        dict_paths=dict_paths,
     )
     return examples or None
 

packages/overture-schema-codegen/src/overture/schema/codegen/markdown_renderer.py

@@ -1,6 +1,7 @@
 """Markdown renderer for Pydantic model documentation."""
 
 import functools
+import json
 import re
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -178,11 +179,11 @@ def _format_example_value(value: object) -> str:
         return f"`{_truncate(value)}`"
 
     if isinstance(value, list):
-        items = ", ".join(repr(item) for item in value)
+        items = ", ".join(json.dumps(item) for item in value)
         return f"`{_truncate(f'[{items}]')}`"
 
     if isinstance(value, dict):
-        pairs = ", ".join(f"{k}: {v}" for k, v in value.items())
+        pairs = ", ".join(f"{json.dumps(k)}: {json.dumps(v)}" for k, v in value.items())
         return f"`{_truncate(f'{{{pairs}}}')}`"
 
     return f"`{value}`"

packages/overture-schema-codegen/src/overture/schema/codegen/reverse_references.py

@@ -85,6 +85,8 @@ def _visit(node: TypeInfo) -> None:
                     referrer_kind,
                 )
 
+            # ENUM, MODEL, pydantic (PRIMITIVE), and UNION are mutually
+            # exclusive by TypeKind.
             if (
                 node.kind in (TypeKind.ENUM, TypeKind.MODEL)
                 and node.source_type is not None
@@ -94,13 +96,11 @@ def _visit(node: TypeInfo) -> None:
                     referrer,
                     referrer_kind,
                 )
-
-            if is_pydantic_type(node):
+            elif is_pydantic_type(node):
                 add_reference(
                     TypeIdentity.of(node.source_type), referrer, referrer_kind
                 )
-
-            if node.union_members is not None:
+            elif node.union_members is not None:
                 for member_cls in node.union_members:
                     add_reference(
                         TypeIdentity.of(member_cls),

packages/overture-schema-codegen/src/overture/schema/codegen/type_collection.py

@@ -87,6 +87,9 @@ def _collect_from_type_info(ti: TypeInfo) -> None:
         """
 
         def _visit(node: TypeInfo) -> None:
+            # UNION, ENUM, and pydantic (PRIMITIVE) are mutually exclusive
+            # by TypeKind. NewType extraction is orthogonal -- a node can be
+            # a NewType-wrapped ENUM, for instance.
             if node.kind == TypeKind.UNION and node.union_members:
                 # Walk each member's fields for supplementary types.
                 # Members that are also top-level feature specs are skipped
@@ -95,11 +98,15 @@ def _visit(node: TypeInfo) -> None:
                     member_spec = extract_model(member_cls)
                     expand_model_tree(member_spec)
                     _collect_from_model(member_spec)
-
-            if node.kind == TypeKind.ENUM and node.source_type is not None:
+            elif node.kind == TypeKind.ENUM and node.source_type is not None:
                 enum_id = TypeIdentity.of(node.source_type)
                 if enum_id not in all_specs:
                     all_specs[enum_id] = extract_enum(node.source_type)
+            elif is_pydantic_type(node):
+                assert node.source_type is not None  # guaranteed by is_pydantic_type
+                pid = TypeIdentity.of(node.source_type)
+                if pid not in all_specs:
+                    all_specs[pid] = extract_pydantic_type(node.source_type)
 
             # Semantic NewTypes always get extracted, including intermediate
             # NewTypes in the wrapping chain (e.g., Id wraps NoWhitespaceString
@@ -115,12 +122,6 @@ def _visit(node: TypeInfo) -> None:
                 if newly_registered:
                     _collect_inner_newtypes(node.newtype_ref)
 
-            if is_pydantic_type(node):
-                assert node.source_type is not None  # guaranteed by is_pydantic_type
-                pid = TypeIdentity.of(node.source_type)
-                if pid not in all_specs:
-                    all_specs[pid] = extract_pydantic_type(node.source_type)
-
         walk_type_info(ti, _visit)
 
     def _collect_from_fields(fields: list[FieldSpec]) -> None:

packages/overture-schema-codegen/src/overture/schema/codegen/union_extraction.py

@@ -76,12 +76,12 @@ def extract_discriminator(
     return disc_field_name, mapping or None
 
 
-_TypeIdentity = tuple[str, TypeKind, bool, int]
-_FieldKey = tuple[str, _TypeIdentity]
+_TypeShape = tuple[str, TypeKind, bool, int]
+_FieldKey = tuple[str, _TypeShape]
 
 
-def _type_identity(ti: TypeInfo) -> _TypeIdentity:
-    """Stable identity for dedup — excludes source_type which can vary across members."""
+def _type_shape(ti: TypeInfo) -> _TypeShape:
+    """Structural shape for dedup -- excludes source_type which varies across members."""
     return (ti.base_type, ti.kind, ti.is_optional, ti.list_depth)
 
 
@@ -117,7 +117,7 @@ def extract_union(
         for fs in member_spec.fields:
             if fs.name in shared_field_names:
                 continue
-            key = (fs.name, _type_identity(fs.type_info))
+            key = (fs.name, _type_shape(fs.type_info))
             existing = seen.get(key)
             prior_sources = existing.variant_sources or () if existing else ()
             seen[key] = AnnotatedField(