feat: semantic layer extension (#37815)

Author: betodealmeida

.github/workflows/superset-python-unittest.yml

@@ -54,6 +54,7 @@ jobs:
           SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov=superset/sql/ ./tests/unit_tests/sql/ --cache-clear --cov-fail-under=100
+          pytest --durations-min=0.5 --cov=superset/semantic_layers/ ./tests/unit_tests/semantic_layers/ --cache-clear --cov-fail-under=100
       - name: Upload code coverage
         uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v5
         with:

UPDATING.md

@@ -46,6 +46,13 @@ The Deck.gl MapBox chart's **Opacity**, **Default longitude**, **Default latitud
 
 **To restore fit-to-data behavior:** Open the chart in Explore, clear the **Default longitude**, **Default latitude**, and **Zoom** fields in the Viewport section, and re-save the chart.
 
+### Combined datasource list endpoint
+
+Added a new combined datasource list endpoint at `GET /api/v1/datasource/` to serve datasets and semantic views in one response.
+
+- The endpoint is available to users with at least one of `can_read` on `Dataset` or `SemanticView`.
+- Semantic views are included only when the `SEMANTIC_LAYERS` feature flag is enabled.
+- The endpoint enforces strict `order_column` validation and returns `400` for invalid sort columns.
 ### ClickHouse minimum driver version bump
 
 The minimum required version of `clickhouse-connect` has been raised to `>=0.13.0`. If you are using the ClickHouse connector, please upgrade your `clickhouse-connect` package. The `_mutate_label` workaround that appended hash suffixes to column aliases has also been removed, as it is no longer needed with modern versions of the driver.

docker/pythonpath_dev/superset_config.py

@@ -105,7 +105,13 @@ class CeleryConfig:
 
 CELERY_CONFIG = CeleryConfig
 
-FEATURE_FLAGS = {"ALERT_REPORTS": True, "DATASET_FOLDERS": True}
+FEATURE_FLAGS = {
+    "ALERT_REPORTS": True,
+    "DATASET_FOLDERS": True,
+    "ENABLE_EXTENSIONS": True,
+    "SEMANTIC_LAYERS": True,
+}
+EXTENSIONS_PATH = "/app/docker/extensions"
 ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
 WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/"  # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/  # noqa: E501
 # The base URL for the email report hyperlinks.

docs/developer_docs/extensions/contribution-types.md

@@ -224,3 +224,52 @@ async def analysis_guide(ctx: Context) -> str:
 ```
 
 See [MCP Integration](./mcp) for implementation details.
+
+### Semantic Layers
+
+Extensions can register custom semantic layer implementations that allow Superset to connect to external data modeling frameworks. Each semantic layer defines how to authenticate, discover semantic views (tables/metrics/dimensions), and execute queries against the external system.
+
+```python
+from superset_core.semantic_layers.decorators import semantic_layer
+from superset_core.semantic_layers.layer import SemanticLayer
+
+from my_extension.config import MyConfig
+from my_extension.view import MySemanticView
+
+
+@semantic_layer(
+    id="my_platform",
+    name="My Data Platform",
+    description="Connect to My Data Platform's semantic layer",
+)
+class MySemanticLayer(SemanticLayer[MyConfig, MySemanticView]):
+    configuration_class = MyConfig
+
+    @classmethod
+    def from_configuration(cls, configuration: dict) -> "MySemanticLayer":
+        config = MyConfig.model_validate(configuration)
+        return cls(config)
+
+    @classmethod
+    def get_configuration_schema(cls, configuration=None) -> dict:
+        return MyConfig.model_json_schema()
+
+    @classmethod
+    def get_runtime_schema(cls, configuration=None, runtime_data=None) -> dict:
+        return {"type": "object", "properties": {}}
+
+    def get_semantic_views(self, runtime_configuration: dict) -> set[MySemanticView]:
+        # Return available views from the external platform
+        ...
+
+    def get_semantic_view(self, name: str, additional_configuration: dict) -> MySemanticView:
+        # Return a specific view by name
+        ...
+```
+
+**Note**: The `@semantic_layer` decorator automatically detects context and applies appropriate ID prefixing:
+
+- **Extension context**: ID prefixed as `extensions.{publisher}.{name}.{id}`
+- **Host context**: Original ID used as-is
+
+The decorator registers the class in the semantic layers registry, making it available in the UI for users to create connections. The `configuration_class` should be a Pydantic model that defines the fields needed to connect (credentials, project, database, etc.). Superset uses the model's JSON schema to render the configuration form dynamically.

docs/static/feature-flags.json

@@ -81,6 +81,12 @@
         "lifecycle": "development",
         "description": "Expand nested types in Presto into extra columns/arrays. Experimental, doesn't work with all nested types."
       },
+      {
+        "name": "SEMANTIC_LAYERS",
+        "default": false,
+        "lifecycle": "development",
+        "description": "Enable semantic layers and show semantic views alongside datasets"
+      },
       {
         "name": "TABLE_V2_TIME_COMPARISON_ENABLED",
         "default": false,

pyproject.toml

@@ -288,6 +288,7 @@ module = [
     "superset.tags.filters",
     "superset.commands.security.update",
     "superset.commands.security.create",
+    "superset.semantic_layers.api",
 ]
 warn_unused_ignores = false
 

superset-core/pyproject.toml

@@ -43,6 +43,8 @@ classifiers = [
 ]
 dependencies = [
     "flask-appbuilder>=5.0.2,<6",
+    "isodate>=0.7.0",
+    "pyarrow>=16.0.0",
     "pydantic>=2.8.0",
     "sqlalchemy>=1.4.0,<2.0",
     "sqlalchemy-utils>=0.38.0, <0.43", # expanding lowerbound to work with pydoris

superset-core/src/superset_core/semantic_layers/config.py

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+def build_configuration_schema(
+    config_class: type[BaseModel],
+    configuration: BaseModel | None = None,
+) -> dict[str, Any]:
+    """
+    Build a JSON schema from a Pydantic configuration class.
+
+    Handles generic boilerplate that any semantic layer with dynamic fields needs:
+
+    - Reorders properties to match model field order (Pydantic sorts alphabetically)
+    - When ``configuration`` is None, sets ``enum: []`` on all ``x-dynamic`` properties
+      so the frontend renders them as empty dropdowns
+
+    Semantic layer implementations call this instead of
+    ``model_json_schema()`` directly,
+    then only need to add their own dynamic population logic.
+    """
+    schema = config_class.model_json_schema()
+
+    # Pydantic sorts properties alphabetically; restore model field order
+    field_order = [
+        field.alias or name for name, field in config_class.model_fields.items()
+    ]
+    schema["properties"] = {
+        key: schema["properties"][key]
+        for key in field_order
+        if key in schema["properties"]
+    }
+
+    if configuration is None:
+        for prop_schema in schema["properties"].values():
+            if prop_schema.get("x-dynamic"):
+                prop_schema["enum"] = []
+
+    return schema
+
+
+def check_dependencies(
+    prop_schema: dict[str, Any],
+    configuration: BaseModel,
+) -> bool:
+    """
+    Check whether a dynamic property's dependencies are satisfied.
+
+    Reads the ``x-dependsOn`` list from the property schema and returns ``True``
+    when every referenced attribute on ``configuration`` is truthy.
+    """
+    dependencies = prop_schema.get("x-dependsOn", [])
+    return all(getattr(configuration, dep, None) for dep in dependencies)

superset-core/src/superset_core/semantic_layers/daos.py

@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Semantic layer DAO interfaces for superset-core.
+
+Provides abstract DAO classes for semantic layers and views that define the
+interface contract. Host implementations replace these with concrete classes
+backed by SQLAlchemy during initialization.
+
+Usage:
+    from superset_core.semantic_layers.daos import (
+        AbstractSemanticLayerDAO,
+        AbstractSemanticViewDAO,
+    )
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Any, ClassVar
+
+from superset_core.common.daos import BaseDAO
+from superset_core.semantic_layers.models import SemanticLayerModel, SemanticViewModel
+
+
+class AbstractSemanticLayerDAO(BaseDAO[SemanticLayerModel]):
+    """
+    Abstract DAO interface for SemanticLayer.
+
+    Host implementations will replace this class during initialization
+    with a concrete DAO providing actual database access.
+    """
+
+    model_cls: ClassVar[type[Any] | None] = None
+    base_filter = None
+    id_column_name = "uuid"
+    uuid_column_name = "uuid"
+
+    @classmethod
+    @abstractmethod
+    def validate_uniqueness(cls, name: str) -> bool:
+        """
+        Validate that a semantic layer name is unique.
+
+        :param name: Semantic layer name to validate
+        :return: True if the name is unique, False otherwise
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def validate_update_uniqueness(cls, layer_uuid: str, name: str) -> bool:
+        """
+        Validate that a semantic layer name is unique for an update operation,
+        excluding the layer being updated.
+
+        :param layer_uuid: UUID of the semantic layer being updated
+        :param name: New name to validate
+        :return: True if the name is unique, False otherwise
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def find_by_name(cls, name: str) -> SemanticLayerModel | None:
+        """
+        Find a semantic layer by name.
+
+        :param name: Semantic layer name
+        :return: SemanticLayerModel instance or None
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def get_semantic_views(cls, layer_uuid: str) -> list[SemanticViewModel]:
+        """
+        Get all semantic views associated with a semantic layer.
+
+        :param layer_uuid: UUID of the semantic layer
+        :return: List of SemanticViewModel instances
+        """
+        ...
+
+
+class AbstractSemanticViewDAO(BaseDAO[SemanticViewModel]):
+    """
+    Abstract DAO interface for SemanticView.
+
+    Host implementations will replace this class during initialization
+    with a concrete DAO providing actual database access.
+    """
+
+    model_cls: ClassVar[type[Any] | None] = None
+    base_filter = None
+    id_column_name = "id"
+    uuid_column_name = "uuid"
+
+    @classmethod
+    @abstractmethod
+    def validate_uniqueness(
+        cls,
+        name: str,
+        layer_uuid: str,
+        configuration: dict[str, Any],
+    ) -> bool:
+        """
+        Validate that a semantic view is unique within a semantic layer.
+
+        Uniqueness is determined by the combination of name, layer UUID, and
+        configuration.
+
+        :param name: View name
+        :param layer_uuid: UUID of the parent semantic layer
+        :param configuration: Configuration dict to compare
+        :return: True if unique, False otherwise
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def validate_update_uniqueness(
+        cls,
+        view_uuid: str,
+        name: str,
+        layer_uuid: str,
+        configuration: dict[str, Any],
+    ) -> bool:
+        """
+        Validate that a semantic view is unique within a semantic layer for an
+        update operation, excluding the view being updated.
+
+        :param view_uuid: UUID of the view being updated
+        :param name: New name to validate
+        :param layer_uuid: UUID of the parent semantic layer
+        :param configuration: Configuration dict to compare
+        :return: True if unique, False otherwise
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def find_by_name(cls, name: str, layer_uuid: str) -> SemanticViewModel | None:
+        """
+        Find a semantic view by name within a semantic layer.
+
+        :param name: View name
+        :param layer_uuid: UUID of the parent semantic layer
+        :return: SemanticViewModel instance or None
+        """
+        ...
+
+
+__all__ = ["AbstractSemanticLayerDAO", "AbstractSemanticViewDAO"]