add superset_openlineage module to superset image

Author: razvan

CHANGELOG.md

@@ -4,11 +4,16 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- superset: Add the `superset-openlineage` integration that emits OpenLineage events for SQL Lab queries and chart/dashboard data fetches; config-only, inert unless enabled in `superset_config.py` ([#XXXX]).
+
 ### Removed
 
 - omid: remove 1.1.2 ([#1593]).
 
 [#1593]: https://github.com/stackabletech/docker-images/pull/1593
+[#XXXX]: https://github.com/stackabletech/docker-images/pull/XXXX
 
 ## [26.7.0] - 2026-07-21
 

superset/Dockerfile

@@ -77,6 +77,7 @@ RUN microdnf update \
 COPY superset/stackable/constraints/${PRODUCT_VERSION}/constraints.txt /tmp/constraints.txt
 COPY superset/stackable/constraints/${PRODUCT_VERSION}/build-constraints.txt /tmp/build-constraints.txt
 COPY --from=opa-authorizer-builder /tmp/opa-authorizer/dist/opa_authorizer-0.1.0-py3-none-any.whl /tmp/
+COPY superset/stackable/superset-openlineage /tmp/superset-openlineage
 
 COPY --chown=${STACKABLE_USER_UID}:0 superset/stackable/patches/patchable.toml /stackable/src/superset/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 superset/stackable/patches/${PRODUCT_VERSION} /stackable/src/superset/stackable/patches/${PRODUCT_VERSION}
@@ -169,6 +170,13 @@ fi
 
 uv pip install --no-cache-dir /tmp/opa_authorizer-0.1.0-py3-none-any.whl
 
+# Install the Stackable OpenLineage integration for Superset. It emits OpenLineage
+# events for SQL Lab queries and chart/dashboard data fetches. It is config-only and
+# stays inert unless enabled in superset_config.py (QUERY_LOGGER, EVENT_LOGGER and
+# init_app via FLASK_APP_MUTATOR); see superset/stackable/superset-openlineage/README.md.
+# sqlglot and SQLAlchemy are provided by Superset itself; only openlineage-python is pulled in.
+uv pip install --no-cache-dir /tmp/superset-openlineage
+
 # Setuptools 82+ removed pkg_resources, which is still needed by Superset 4.x
 # dependencies. Re-pin after all other installs in case newer versions
 # have been pulled in by other dependencies.

superset/stackable/superset-openlineage/README.md

@@ -0,0 +1,76 @@
+# superset-openlineage
+
+OpenLineage integration for Apache Superset, shipped in the Stackable Superset
+image. It emits OpenLineage events when SQL Lab queries run and when dashboard
+charts fetch data.
+
+It is **config-only**: no changes are made to the Superset source tree, and the
+plugin stays completely inert unless it is explicitly enabled in
+`superset_config.py`. The package is installed into the image (see the
+`superset/Dockerfile` build stage), but activation is left to the operator/admin.
+
+## Enable it
+
+Add the following to `superset_config.py`:
+
+```python
+from superset_openlineage import (
+    build_query_logger,
+    OpenLineageEventLogger,
+    init_app,
+)
+
+QUERY_LOGGER = build_query_logger()
+EVENT_LOGGER = OpenLineageEventLogger()
+
+# Registers a teardown_request that clears the request-scoped stash
+# (required to avoid cross-request lineage mis-attribution on reused workers).
+def FLASK_APP_MUTATOR(app):
+    init_app(app)
+```
+
+## Configure the transport
+
+Point the client at a lineage backend with standard OpenLineage configuration
+(environment variables or `openlineage.yml`):
+
+```bash
+export OPENLINEAGE_URL=http://marquez:5000
+export OPENLINEAGE_API_KEY=<token>        # optional
+# or set OPENLINEAGE_DISABLED=true to turn emission off entirely
+```
+
+## Plugin settings
+
+| Env var | Default | Meaning |
+|---|---|---|
+| `SUPERSET_OPENLINEAGE_ENABLED` | `true` | Master switch for the plugin |
+| `SUPERSET_OPENLINEAGE_NAMESPACE` | `superset` | OpenLineage job namespace |
+| `SUPERSET_OPENLINEAGE_PRODUCER` | Superset repo URL | `producer` URI on events |
+| `SUPERSET_OPENLINEAGE_JOB_NAME` | `$source.$identity` | Job-name template (`string.Template`) |
+
+### Job-name template
+
+A Python `string.Template`. Placeholders: `$source`, `$identity`,
+`$sql_editor_id`, `$client_id`, `$slice_id`, `$dashboard_id`, `$sql_hash`,
+`$schema`, `$catalog`, `$database`, `$username`. Placeholders that do not apply
+in a given context render empty and surrounding separators collapse; an empty
+result falls back to `$source.$sql_hash`.
+
+## Known limitations
+
+- Failed queries and cache hits emit no events (the config-only hooks provide no
+  failure or cache signal).
+- Column-level lineage is emitted only for `CTAS`/`CVAS` statements (which have a
+  real output table); plain `SELECT` queries (charts/dashboards and most SQL Lab)
+  carry table-level inputs only.
+- Queries executed via Superset's newer SQL execution engine
+  (`superset/sql/execution/*`) are not covered yet — they invoke `QUERY_LOGGER`
+  but emit no completion event, so they produce no lineage (their pending stash
+  entries are reclaimed by the teardown hook).
+
+## Tests
+
+Unit tests live under `superset_openlineage/tests/` and require
+`openlineage-python`, `sqlglot`, `SQLAlchemy` and `pytest`. They are not
+installed into the image.

superset/stackable/superset-openlineage/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools>=64"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "superset-openlineage"
+version = "0.1.0"
+description = "OpenLineage integration for Apache Superset"
+# Keep in sync with the python-version build arguments in superset/boil-config.toml.
+requires-python = ">=3.10"
+# Only openlineage-python is pulled in here. sqlglot and SQLAlchemy are runtime
+# dependencies of Apache Superset itself, so they are intentionally NOT declared
+# to avoid conflicting with Superset's own pins.
+dependencies = [
+    "openlineage-python>=1.30,<2",
+]
+
+[tool.setuptools]
+# Only the package is installed; the tests/ subpackage is intentionally excluded
+# from the wheel.
+packages = ["superset_openlineage"]

superset/stackable/superset-openlineage/superset_openlineage/__init__.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""OpenLineage integration for Apache Superset (config-only plugin)."""
+
+__all__ = ["build_query_logger", "OpenLineageEventLogger", "init_app"]
+
+
+# lazy re-export to avoid import cost at config load
+def __getattr__(name: str) -> object:
+    if name == "build_query_logger":
+        from superset_openlineage.query_logger import build_query_logger
+
+        return build_query_logger
+    if name == "OpenLineageEventLogger":
+        from superset_openlineage.event_logger import OpenLineageEventLogger
+
+        return OpenLineageEventLogger
+    if name == "init_app":
+        from superset_openlineage.lifecycle import init_app
+
+        return init_app
+    raise AttributeError(name)

superset/stackable/superset-openlineage/superset_openlineage/client.py

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from superset_openlineage import settings
+
+logger = logging.getLogger(__name__)
+
+_client: Any = None
+_initialized: bool = False
+
+
+def reset_client() -> None:
+    global _client, _initialized
+    _client = None
+    _initialized = False
+
+
+def get_client() -> Any:
+    global _client, _initialized
+    if _initialized:
+        return _client
+    _initialized = True
+    if not settings.is_enabled():
+        _client = None
+        return None
+    try:
+        from openlineage.client import OpenLineageClient
+
+        _client = OpenLineageClient()
+    except Exception as ex:  # noqa: BLE001 - never break Superset on OL init
+        logger.warning("OpenLineage: client init failed, disabling: %s", ex)
+        _client = None
+    return _client
+
+
+def emit(event: Any) -> None:
+    client = get_client()
+    if client is None:
+        return
+    try:
+        client.emit(event)
+    except Exception as ex:  # noqa: BLE001 - emission must never break a query
+        logger.warning("OpenLineage: emit failed, dropping event: %s", ex)

superset/stackable/superset-openlineage/superset_openlineage/context.py

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from typing import Any
+
+_PENDING: ContextVar[list["PendingRun"] | None] = ContextVar("ol_pending", default=None)
+
+
+@dataclass
+class PendingRun:
+    run_id: str
+    sql: str
+    dialect: str | None
+    source: str
+    sql_hash: str
+    schema: str | None
+    catalog: str | None
+    database: str | None
+    username: str | None
+    sql_editor_id: str | None
+    client_id: str | None
+    start_time: float
+    inputs: list[Any] = field(default_factory=list)
+    outputs: list[Any] = field(default_factory=list)
+    column_lineage: Any = None
+
+
+def _stack() -> list[PendingRun]:
+    stack = _PENDING.get()
+    if stack is None:
+        stack = []
+        _PENDING.set(stack)
+    return stack
+
+
+def push_pending(run: PendingRun) -> None:
+    stack = list(_stack())
+    stack.append(run)
+    _PENDING.set(stack)
+
+
+def pop_pending() -> PendingRun | None:
+    stack = list(_stack())
+    if not stack:
+        return None
+    run = stack.pop()
+    _PENDING.set(stack)
+    return run
+
+
+def clear_pending() -> None:
+    _PENDING.set([])