refactor(system): tighten discovery api and various small improvements

Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com>

Author: RoelBollens-TomTom

packages/overture-schema-cli/src/overture/schema/cli/__init__.py

@@ -11,7 +11,6 @@
 )
 from .types import (
     ErrorLocation,
-    ModelDict,
     UnionType,
     ValidationErrorDict,
 )
@@ -25,7 +24,6 @@
     "perform_validation",
     "resolve_types",
     "ErrorLocation",
-    "ModelDict",
     "UnionType",
     "ValidationErrorDict",
 ]

packages/overture-schema-cli/src/overture/schema/cli/commands.py

@@ -22,8 +22,9 @@
     ModelKey,
     discover_models,
     filter_models,
-    tags_by_key,
 )
+from overture.schema.system.discovery.tag import get_values_for_key
+from overture.schema.system.discovery.types import ModelDict
 from overture.schema.system.feature import Feature
 from overture.schema.system.json_schema import json_schema
 
@@ -34,7 +35,7 @@
     select_most_likely_errors,
 )
 from .type_analysis import StructuralTuple, get_item_index, introspect_union
-from .types import ErrorLocation, ModelDict, UnionType
+from .types import ErrorLocation, UnionType
 
 # Console instances for rich output
 stdout = Console(highlight=False)
@@ -214,7 +215,9 @@ def resolve_types(
     models: ModelDict = discover_models()
 
     # Filter models based on CLI options
-    models = filter_models(models, tags, excluded_tags, type_names)
+    models = filter_models(
+        models, tags=tags, excluded_tags=excluded_tags, type_names=type_names
+    )
 
     if not models:
         raise ValueError("No models found matching the specified criteria")
@@ -749,14 +752,14 @@ def json_schema_command(
 )
 @click.option(
     "--group-by",
-    help="Group types by tag prefix (e.g., 'overture:theme')",
+    help="Group types by tag key (e.g., 'overture:theme')",
 )
 def list_types(
     tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None
 ) -> None:
-    r"""List all available types grouped by theme with descriptions.
+    r"""List all available types.
 
-    Displays all registered Overture Maps types and can organized by grouping.
+    Displays all registered models and can be organized by grouping.
 
     \b
     Examples:
@@ -772,7 +775,7 @@ def list_types(
             grouped_models: dict[str, set[ModelKey]] = {}
 
             for key in models.keys():
-                if groups := tags_by_key(key.tags, group_by):
+                if groups := get_values_for_key(key.tags, group_by):
                     for group in groups:
                         grouped_models.setdefault(group, set()).add(key)
 

packages/overture-schema-cli/src/overture/schema/cli/types.py

@@ -5,15 +5,10 @@
 from pydantic import BaseModel
 from pydantic_core import ErrorDetails
 
-from overture.schema.system.discovery import ModelKey
-
 # Type alias for union types created from Pydantic models
 # This represents either a single model or a discriminated union of models
 UnionType: TypeAlias = type[BaseModel] | Any
 
-# Dictionary mapping ModelKey to Pydantic model classes
-ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]]
-
 # Pydantic validation error dictionary structure
 # In Pydantic v2, ValidationError.errors() returns list[ErrorDetails]
 ValidationErrorDict: TypeAlias = ErrorDetails

packages/overture-schema-codegen/README.md

@@ -22,7 +22,7 @@ renderers, not extraction logic.
 overture-codegen generate --format markdown --output-dir docs/schema/reference
 
 # Generate for a single theme
-overture-codegen generate --format markdown --theme buildings --output-dir out/
+overture-codegen generate --format markdown --tag overture:theme=buildings --output-dir out/
 
 # List discovered models
 overture-codegen list

packages/overture-schema-codegen/tests/codegen_test_support.py

@@ -25,7 +25,8 @@
     is_model_class,
 )
 from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind
-from overture.schema.system.discovery import discover_models, tags_by_key
+from overture.schema.system.discovery import discover_models, filter_models
+from overture.schema.system.discovery.tag import get_values_for_key
 from overture.schema.system.doc import DocumentedEnum
 from overture.schema.system.field_constraint import UniqueItemsConstraint
 from overture.schema.system.model_constraint import require_any_of
@@ -303,7 +304,7 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec:
 
 def find_theme(tags: frozenset[str]) -> str | None:
     """Extract the theme from a set of tags, if present."""
-    return next(iter(tags_by_key(tags, "overture:theme")), None)
+    return next(iter(get_values_for_key(tags, "overture:theme")), None)
 
 
 T = TypeVar("T")
@@ -337,7 +338,7 @@ def flat_specs_from_discovery(
     """Build a flat list of ModelSpecs from discovery, with entry_point set."""
     models = discover_models()
     if theme:
-        models = {k: v for k, v in models.items() if find_theme(k.tags) == theme}
+        models = filter_models(models, tags=(f"overture:theme={theme}",))
     result = []
     for key, cls in models.items():
         if not is_model_class(cls):

packages/overture-schema-core/src/overture/schema/core/tag_providers.py

@@ -1,9 +1,10 @@
-from typing import Annotated, Any, Literal, Union, get_args, get_origin
+from typing import get_args
 
 from pydantic import BaseModel
 
 from overture.schema.core import OvertureFeature
 from overture.schema.system.discovery import ModelKey
+from overture.schema.system.typing_util import collect_types
 
 APPROVED = {
     "overture.schema.addresses:Address",
@@ -28,6 +29,22 @@
 def authority_provider(
     model_class: type[BaseModel], key: ModelKey, tags: set[str]
 ) -> set[str]:
+    """Add the ``"overture"`` tag if the model originates from an approved Overture package.
+
+    Parameters
+    ----------
+    model_class : type[BaseModel]
+        Model class to inspect.
+    key : ModelKey
+        Key identifying the model.
+    tags : set[str]
+        Current tags; may be extended.
+
+    Returns
+    -------
+    set[str]
+        Updated tags, with ``"overture"`` added if applicable.
+    """
     if _matches_manifest(key):
         tags.add("overture")
     return tags
@@ -36,8 +53,24 @@ def authority_provider(
 def theme_provider(
     model_class: type[BaseModel], key: ModelKey, tags: set[str]
 ) -> set[str]:
-    for tp in _extract_types(model_class):
-        if isinstance(tp, type) and issubclass(tp, OvertureFeature):
+    """Add the ``"overture:theme={theme}"`` tag if the model is a subclass of OvertureFeature.
+
+    Parameters
+    ----------
+    model_class : type[BaseModel]
+        Model class to inspect.
+    key : ModelKey
+        Key identifying the model.
+    tags : set[str]
+        Current tags; may be extended.
+
+    Returns
+    -------
+    set[str]
+        Updated tags, with ``"overture:theme={theme}"`` added if applicable.
+    """
+    for tp in collect_types(model_class):
+        if issubclass(tp, OvertureFeature):
             tags.add(
                 "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0]
             )
@@ -46,34 +79,3 @@ def theme_provider(
 
 def _matches_manifest(key: ModelKey) -> bool:
     return key.entry_point in APPROVED
-
-
-def _extract_types(tp: Any) -> set[type]:  # noqa: ANN401
-    result: set[type] = set()
-
-    def visit(t: Any) -> None:  # noqa: ANN401
-        origin = get_origin(t)
-        if origin is Annotated:
-            visit(get_args(t)[0])
-            return
-
-        if hasattr(t, "__supertype__"):
-            visit(t.__supertype__)
-            return
-
-        origin = get_origin(t)
-
-        if origin is Union:
-            for arg in get_args(t):
-                visit(arg)
-            return
-
-        if origin is Literal:
-            for val in get_args(t):
-                result.add(type(val))
-            return
-
-        result.add(t)
-
-    visit(tp)
-    return result

packages/overture-schema-core/tests/test_approved_models.py

@@ -4,7 +4,7 @@
 def test_overture_feature_models_are_official() -> None:
     models = discover_models()
     for key in models:
-        if "overture:feature" in key.tags:
-            assert "overture:official" in key.tags, (
+        if "feature" in key.tags:
+            assert "overture" in key.tags, (
                 f"Model {key.name} is missing 'overture:official' tag."
             )

packages/overture-schema-system/pyproject.toml

@@ -57,4 +57,4 @@ ignore = [
 per-file-ignores = {"__init__.py" = ["F401"]}
 
 [project.entry-points."overture.tag_providers"]
-feature = "overture.schema.system.discovery:feature_provider"
+feature = "overture.schema.system.discovery.tag_providers:feature_provider"