From c0c8e05537c634086d7f240d1eac8e6f4a44064e Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 3 Mar 2026 15:04:20 -0800 Subject: [PATCH 01/12] Add experimental labeler in otel context --- .../instrumentation/_labeler/__init__.py | 92 ++++++ .../_labeler/_internal/__init__.py | 279 ++++++++++++++++++ .../tests/test_labeler.py | 210 +++++++++++++ pyproject.toml | 1 + 4 files changed, 582 insertions(+) create mode 100644 opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py create mode 100644 opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/_internal/__init__.py create mode 100644 opentelemetry-instrumentation/tests/test_labeler.py diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py new file mode 100644 index 0000000000..621f4f6965 --- /dev/null +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py @@ -0,0 +1,92 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenTelemetry Labeler +===================== + +The labeler utility provides a way to add custom attributes to some metrics +generated by OpenTelemetry instrumentations. + +This was inspired by OpenTelemetry Go's net/http instrumentation Labeler +https://github.com/open-telemetry/opentelemetry-go-contrib/pull/306 + +Usage +----- + +The labeler is typically used within the context of an instrumented request +or operation. Use ``get_labeler`` to obtain a labeler instance for the +current context, then add attributes using the ``add`` or +``add_attributes`` methods. + +Example with Flask +------------------ + +Here's an example showing how to use the labeler with programmatic Flask instrumentation: + +.. code-block:: python + + from flask import Flask + from opentelemetry.instrumentation._labeler import get_labeler + from opentelemetry.instrumentation.flask import FlaskInstrumentor + + app = Flask(__name__) + FlaskInstrumentor().instrument_app(app) + + @app.route("/healthcheck") + def healthcheck(): + return "OK" + + @app.route("/user/") + def user_profile(user_id): + labeler = get_labeler() + + # Can add individual attributes or multiple at once + labeler.add("user_id", user_id) + labeler.add_attributes( + { + "has_premium": user_id in ["123", "456"], + "experiment_group": "control", + "feature_enabled": True, + "user_segment": "active", + } + ) + + return f"Got user profile for {user_id}" + +The labeler can also be used with auto-instrumentation. + +Custom attributes are merged by instrumentors that integrate +``enrich_metric_attributes`` before recording metrics (for example, +``Histogram.record``). ``enrich_metric_attributes`` does not overwrite base +attributes that exist at the same keys. +""" + +from opentelemetry.instrumentation._labeler._internal import ( + Labeler, + clear_labeler, + enrich_metric_attributes, + get_labeler, + get_labeler_attributes, + set_labeler, +) + +__all__ = [ + "Labeler", + "get_labeler", + "set_labeler", + "clear_labeler", + "get_labeler_attributes", + "enrich_metric_attributes", +] diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/_internal/__init__.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/_internal/__init__.py new file mode 100644 index 0000000000..0230d6de0a --- /dev/null +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/_internal/__init__.py @@ -0,0 +1,279 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading +from types import MappingProxyType +from typing import Any, Dict, Mapping, Optional, Union + +from opentelemetry.context import attach, create_key, get_value, set_value +from opentelemetry.util.types import AttributeValue + +LABELER_CONTEXT_KEY = create_key("otel_labeler") + +_logger = logging.getLogger(__name__) + + +class Labeler: + """ + Stores custom attributes for the current OTel context. + + This feature is experimental and unstable. + """ + + def __init__( + self, max_custom_attrs: int = 20, max_attr_value_length: int = 100 + ): + """ + Initialize a new Labeler instance. + + Args: + max_custom_attrs: Maximum number of custom attributes to store. + When this limit is reached, new attributes will be ignored; + existing attributes can still be updated. + max_attr_value_length: Maximum length for string attribute values. + String values exceeding this length will be truncated. + """ + self._lock = threading.Lock() + self._attributes: Dict[str, Union[str, int, float, bool]] = {} + self._max_custom_attrs = max_custom_attrs + self._max_attr_value_length = max_attr_value_length + + def add(self, key: str, value: Any) -> None: + """ + Add a single attribute to the labeler, subject to the labeler's limits: + - If max_custom_attrs limit is reached and this is a new key, the attribute is ignored + - String values exceeding max_attr_value_length are truncated + + Args: + key: attribute key + value: attribute value, must be a primitive type: str, int, float, or bool + """ + if not isinstance(value, (str, int, float, bool)): + _logger.warning( + "Skipping attribute '%s': value must be str, int, float, or bool, got %s", + key, + type(value).__name__, + ) + return + + with self._lock: + if ( + len(self._attributes) >= self._max_custom_attrs + and key not in self._attributes + ): + return + + if ( + isinstance(value, str) + and len(value) > self._max_attr_value_length + ): + value = value[: self._max_attr_value_length] + + self._attributes[key] = value + + def add_attributes(self, attributes: Dict[str, Any]) -> None: + """ + Add multiple attributes to the labeler, subject to the labeler's limits: + - If max_custom_attrs limit is reached and this is a new key, the attribute is ignored + - Existing attributes can still be updated + - String values exceeding max_attr_value_length are truncated + + Args: + attributes: Dictionary of attributes to add. Values must be primitive types + (str, int, float, or bool) + """ + with self._lock: + for key, value in attributes.items(): + if not isinstance(value, (str, int, float, bool)): + _logger.warning( + "Skipping attribute '%s': value must be str, int, float, or bool, got %s", + key, + type(value).__name__, + ) + continue + + if ( + len(self._attributes) >= self._max_custom_attrs + and key not in self._attributes + ): + continue + + if ( + isinstance(value, str) + and len(value) > self._max_attr_value_length + ): + value = value[: self._max_attr_value_length] + + self._attributes[key] = value + + def get_attributes(self) -> Mapping[str, Union[str, int, float, bool]]: + """ + Return a read-only mapping view of attributes in this labeler. + """ + with self._lock: + return MappingProxyType(self._attributes) + + def clear(self) -> None: + with self._lock: + self._attributes.clear() + + def __len__(self) -> int: + with self._lock: + return len(self._attributes) + + +def _attach_context_value(value: Optional[Labeler]) -> None: + """ + Attach a new OpenTelemetry context containing the given labeler value. + + This helper is fail-safe: context attach errors are suppressed and + logged at debug level. + + Args: + value: Labeler instance to store in context, or ``None`` to clear it. + """ + try: + updated_context = set_value(LABELER_CONTEXT_KEY, value) + attach(updated_context) + except Exception: # pylint: disable=broad-exception-caught + _logger.debug("Failed to attach labeler context", exc_info=True) + + +def get_labeler() -> Labeler: + """ + Get the Labeler instance for the current OTel context. + + If no Labeler exists in the current context, a new one is created + and stored in the context. + + Returns: + Labeler instance for the current OTel context, or a new empty Labeler + if no Labeler is currently stored in context. + """ + try: + current_value = get_value(LABELER_CONTEXT_KEY) + except Exception: # pylint: disable=broad-exception-caught + _logger.debug("Failed to read labeler from context", exc_info=True) + current_value = None + + if isinstance(current_value, Labeler): + return current_value + + labeler = Labeler() + _attach_context_value(labeler) + return labeler + + +def set_labeler(labeler: Any) -> None: + """ + Set the Labeler instance for the current OTel context. + + Args: + labeler: The Labeler instance to set + """ + if not isinstance(labeler, Labeler): + _logger.warning( + "Skipping set_labeler: value must be Labeler, got %s", + type(labeler).__name__, + ) + return + _attach_context_value(labeler) + + +def clear_labeler() -> None: + """ + Clear the Labeler instance from the current OTel context. + + This is primarily intended for test isolation or manual context-lifecycle + management. In typical framework-instrumented request handling, + applications generally should not need to call this directly. + """ + _attach_context_value(None) + + +def get_labeler_attributes() -> Mapping[str, Union[str, int, float, bool]]: + """ + Get attributes from the current labeler, if any. + + Returns: + Read-only mapping of custom attributes, or an empty read-only mapping + if no labeler exists. + """ + empty_attributes: Dict[str, Union[str, int, float, bool]] = {} + try: + current_value = get_value(LABELER_CONTEXT_KEY) + except Exception: # pylint: disable=broad-exception-caught + _logger.debug( + "Failed to read labeler attributes from context", exc_info=True + ) + return MappingProxyType(empty_attributes) + + if not isinstance(current_value, Labeler): + return MappingProxyType(empty_attributes) + return current_value.get_attributes() + + +def enrich_metric_attributes( + base_attributes: Dict[str, Any], + enrich_enabled: bool = True, +) -> Dict[str, AttributeValue]: + """ + Combines base_attributes with custom attributes from the current labeler, + returning a new dictionary of attributes according to the labeler configuration: + - Attributes that would override base_attributes are skipped + - If max_custom_attrs limit is reached and this is a new key, the attribute is ignored + - String values exceeding max_attr_value_length are truncated + + Args: + base_attributes: The base attributes for the metric + enrich_enabled: Whether to include custom labeler attributes + + Returns: + Dictionary combining base and custom attributes. If no custom attributes, + returns a copy of the original base attributes. + """ + if not enrich_enabled: + return base_attributes.copy() + + labeler_attributes = get_labeler_attributes() + if not labeler_attributes: + return base_attributes.copy() + + try: + labeler = get_value(LABELER_CONTEXT_KEY) + except Exception: # pylint: disable=broad-exception-caught + labeler = None + + if not isinstance(labeler, Labeler): + return base_attributes.copy() + + enriched_attributes = base_attributes.copy() + added_count = 0 + for key, value in labeler_attributes.items(): + if added_count >= labeler._max_custom_attrs: + break + if key in base_attributes: + continue + + if ( + isinstance(value, str) + and len(value) > labeler._max_attr_value_length + ): + value = value[: labeler._max_attr_value_length] + + enriched_attributes[key] = value + added_count += 1 + + return enriched_attributes diff --git a/opentelemetry-instrumentation/tests/test_labeler.py b/opentelemetry-instrumentation/tests/test_labeler.py new file mode 100644 index 0000000000..673cba1cc0 --- /dev/null +++ b/opentelemetry-instrumentation/tests/test_labeler.py @@ -0,0 +1,210 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextvars +import threading +import unittest +from unittest.mock import patch + +from opentelemetry.instrumentation._labeler import ( + Labeler, + clear_labeler, + enrich_metric_attributes, + get_labeler, + get_labeler_attributes, + set_labeler, +) +from opentelemetry.instrumentation._labeler._internal import ( + LABELER_CONTEXT_KEY, +) + + +class TestLabeler(unittest.TestCase): + def setUp(self): + clear_labeler() + + def test_labeler_init(self): + labeler = Labeler() + self.assertEqual(labeler.get_attributes(), {}) + self.assertEqual(len(labeler), 0) + + def test_add_single_attribute(self): + labeler = Labeler() + labeler.add("test_key", "test_value") + self.assertEqual(labeler.get_attributes(), {"test_key": "test_value"}) + + def test_add_attributes_dict(self): + labeler = Labeler() + attrs = {"key1": "value1", "key2": 42, "key3": False} + labeler.add_attributes(attrs) + self.assertEqual(labeler.get_attributes(), attrs) + + def test_overwrite_attribute(self): + labeler = Labeler() + labeler.add("key1", "original") + labeler.add("key1", "updated") + self.assertEqual(labeler.get_attributes(), {"key1": "updated"}) + + def test_clear_attributes(self): + labeler = Labeler() + labeler.add("key1", "value1") + labeler.add("key2", "value2") + labeler.clear() + self.assertEqual(labeler.get_attributes(), {}) + + def test_add_invalid_types_logs_warning_and_skips(self): + labeler = Labeler() + with patch( + "opentelemetry.instrumentation._labeler._internal._logger.warning" + ) as mock_warning: + labeler.add("valid", "value") + labeler.add("dict_key", {"nested": "dict"}) + labeler.add("list_key", [1, 2, 3]) + labeler.add("none_key", None) + labeler.add("another_valid", 123) + + self.assertEqual(mock_warning.call_count, 3) + self.assertEqual( + labeler.get_attributes(), {"valid": "value", "another_valid": 123} + ) + + def test_limit_and_truncation(self): + labeler = Labeler(max_custom_attrs=2, max_attr_value_length=5) + labeler.add("a", "1234567") + labeler.add("b", "ok") + labeler.add("c", "ignored") + self.assertEqual(labeler.get_attributes(), {"a": "12345", "b": "ok"}) + + def test_enrich_metric_attributes_skips_base_overrides(self): + base_attributes = {"http.method": "GET", "http.status_code": 200} + labeler = get_labeler() + labeler.add("http.method", "POST") + labeler.add("custom_attr", "test-value") + + enriched = enrich_metric_attributes(base_attributes) + self.assertEqual(enriched["http.method"], "GET") + self.assertEqual(enriched["custom_attr"], "test-value") + self.assertEqual(base_attributes["http.method"], "GET") + + def test_thread_safety(self): + labeler = Labeler(max_custom_attrs=1100) + num_threads = 5 + num_ops = 100 + + def worker(thread_id): + for index in range(num_ops): + labeler.add(f"thread_{thread_id}_{index}", f"v_{index}") + + threads = [ + threading.Thread(target=worker, args=(i,)) + for i in range(num_threads) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + self.assertEqual(len(labeler.get_attributes()), num_threads * num_ops) + + +class TestLabelerContext(unittest.TestCase): + def setUp(self): + clear_labeler() + + def test_get_labeler_creates_new(self): + labeler = get_labeler() + self.assertIsInstance(labeler, Labeler) + + def test_get_labeler_returns_same_instance(self): + labeler1 = get_labeler() + labeler1.add("test", "value") + labeler2 = get_labeler() + self.assertIs(labeler1, labeler2) + + def test_set_labeler(self): + custom_labeler = Labeler() + custom_labeler.add("custom", "value") + set_labeler(custom_labeler) + self.assertIs(get_labeler(), custom_labeler) + + def test_set_labeler_invalid_type_is_ignored(self): + with patch( + "opentelemetry.instrumentation._labeler._internal._logger.warning" + ) as mock_warning: + set_labeler("bad") # type: ignore[arg-type] + self.assertEqual(mock_warning.call_count, 1) + + def test_clear_labeler(self): + labeler = get_labeler() + labeler.add("test", "value") + clear_labeler() + new_labeler = get_labeler() + self.assertIsNot(new_labeler, labeler) + self.assertEqual(new_labeler.get_attributes(), {}) + + def test_get_labeler_attributes(self): + clear_labeler() + self.assertEqual(get_labeler_attributes(), {}) + labeler = get_labeler() + labeler.add("test", "value") + self.assertEqual(get_labeler_attributes(), {"test": "value"}) + + def test_context_isolation(self): + def context_worker(context_id, results): + labeler = get_labeler() + labeler.add("context_id", context_id) + results[context_id] = dict(labeler.get_attributes()) + + results = {} + for operation in range(3): + ctx = contextvars.copy_context() + ctx.run(context_worker, operation, results) + + for operation in range(3): + self.assertEqual(results[operation], {"context_id": operation}) + + +class TestLabelerFailSafe(unittest.TestCase): + def setUp(self): + clear_labeler() + + def test_get_labeler_failsafe_on_get_value_error(self): + with patch( + "opentelemetry.instrumentation._labeler._internal.get_value", + side_effect=RuntimeError("boom"), + ): + labeler = get_labeler() + self.assertIsInstance(labeler, Labeler) + + def test_set_and_clear_failsafe_on_attach_error(self): + labeler = Labeler() + with patch( + "opentelemetry.instrumentation._labeler._internal.attach", + side_effect=RuntimeError("boom"), + ): + set_labeler(labeler) + clear_labeler() + + self.assertIsInstance(get_labeler(), Labeler) + + def test_get_labeler_attributes_failsafe(self): + with patch( + "opentelemetry.instrumentation._labeler._internal.get_value", + side_effect=RuntimeError("boom"), + ): + attrs = get_labeler_attributes() + self.assertEqual(attrs, {}) + + def test_context_key_constant(self): + self.assertTrue(isinstance(LABELER_CONTEXT_KEY, str)) diff --git a/pyproject.toml b/pyproject.toml index 99ba8d2e32..eeea47e0c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,6 +209,7 @@ include = [ "util/opentelemetry-util-genai", "exporter/opentelemetry-exporter-credential-provider-gcp", "instrumentation/opentelemetry-instrumentation-aiohttp-client", + "opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler", ] # We should also add type hints to the test suite - It helps on finding bugs. # We are excluding for now because it's easier, and more important to add to the instrumentation packages. From 17757c386720e37701b9368862c6dfe0e2fd03da Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 3 Mar 2026 16:00:56 -0800 Subject: [PATCH 02/12] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb7aefc82..33ed146ed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#4139](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4139)) - `opentelemetry-instrumentation-logging`: Move there the SDK LoggingHandler ([#4210](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4210)) +- `opentelemetry-instrumentation`: Add experimental attributes Labeler utility + ([#4288](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4288)) ### Fixed From dbfa67cf744c9fe08e82f05285f22e481ac84ba2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Mar 2026 16:20:31 -0800 Subject: [PATCH 03/12] Adjust wording --- CHANGELOG.md | 2 +- .../instrumentation/_labeler/__init__.py | 54 ++++++++++--------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3a622a6a..e53d62c877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `opentelemetry-instrumentation`: Add experimental attributes Labeler utility +- `opentelemetry-instrumentation`: Add experimental metrics attributes Labeler utility ([#4288](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4288)) ## Version 1.40.0/0.61b0 (2026-03-04) diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py index 621f4f6965..e36fe18733 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/_labeler/__init__.py @@ -16,8 +16,7 @@ OpenTelemetry Labeler ===================== -The labeler utility provides a way to add custom attributes to some metrics -generated by OpenTelemetry instrumentations. +The labeler utility provides a way to add custom attributes to metrics. This was inspired by OpenTelemetry Go's net/http instrumentation Labeler https://github.com/open-telemetry/opentelemetry-go-contrib/pull/306 @@ -30,29 +29,28 @@ current context, then add attributes using the ``add`` or ``add_attributes`` methods. -Example with Flask ------------------- +Example +------- -Here's an example showing how to use the labeler with programmatic Flask instrumentation: +Here's a framework-agnostic example showing manual use of the labeler: .. code-block:: python - from flask import Flask - from opentelemetry.instrumentation._labeler import get_labeler - from opentelemetry.instrumentation.flask import FlaskInstrumentor + from opentelemetry.instrumentation._labeler import ( + enrich_metric_attributes, + get_labeler, + ) + from opentelemetry.metrics import get_meter - app = Flask(__name__) - FlaskInstrumentor().instrument_app(app) + meter = get_meter("example.manual") + duration_histogram = meter.create_histogram( + name="http.server.request.duration", + unit="s", + description="Duration of HTTP server requests.", + ) - @app.route("/healthcheck") - def healthcheck(): - return "OK" - - @app.route("/user/") - def user_profile(user_id): + def record_request(user_id: str, duration_s: float) -> None: labeler = get_labeler() - - # Can add individual attributes or multiple at once labeler.add("user_id", user_id) labeler.add_attributes( { @@ -63,14 +61,22 @@ def user_profile(user_id): } ) - return f"Got user profile for {user_id}" + base_attributes = { + "http.request.method": "GET", + "http.response.status_code": 200, + } + duration_histogram.record( + max(duration_s, 0), + enrich_metric_attributes(base_attributes), + ) -The labeler can also be used with auto-instrumentation. +This package introduces the shared Labeler API and helper utilities. +Framework-specific integration points that call +``enrich_metric_attributes`` (for example before ``Histogram.record``) +can be added by individual instrumentors. -Custom attributes are merged by instrumentors that integrate -``enrich_metric_attributes`` before recording metrics (for example, -``Histogram.record``). ``enrich_metric_attributes`` does not overwrite base -attributes that exist at the same keys. +When instrumentors use ``enrich_metric_attributes``, it does not +overwrite base attributes that exist at the same keys. """ from opentelemetry.instrumentation._labeler._internal import ( From 2ce042f8f448980ff0eeab17caf1e750679c8280 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Mar 2026 17:39:37 -0800 Subject: [PATCH 04/12] Add WSGI,ASGI enrich_metric_attributes HTTP server duration --- .../instrumentation/asgi/__init__.py | 66 ++++++++++++++++ .../tests/test_asgi_middleware.py | 45 +++++++++++ .../instrumentation/wsgi/__init__.py | 78 ++++++++++++++++++- .../tests/test_wsgi_middleware.py | 41 ++++++++++ 4 files changed, 227 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py index 59c5083ada..a022ceb5b2 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py @@ -204,6 +204,46 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]): Note: The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. +Custom Metrics Attributes using Labeler +*************************************** +The ASGI instrumentation reads from a labeler utility that supports adding custom +attributes to HTTP duration metrics at record time. The custom attributes are +stored only within the context of an instrumented request or operation. The +instrumentor does not overwrite base attributes that exist at the same keys as +any custom attributes. + + +.. code-block:: python + + .. code-block:: python + + from quart import Quart + from opentelemetry.instrumentation._labeler import get_labeler + from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware + + app = Quart(__name__) + app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) + + @app.route("/users//") + async def user_profile(user_id): + # Get the labeler for the current request + labeler = get_labeler() + + # Add custom attributes to ASGI instrumentation metrics + labeler.add("user_id", user_id) + labeler.add("user_type", "registered") + + # Or, add multiple attributes at once + labeler.add_attributes({ + "feature_flag": "new_ui", + "experiment_group": "control" + }) + + return f"User profile for {user_id}" + + if __name__ == "__main__": + app.run(debug=True) + API --- """ @@ -220,6 +260,11 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]): from asgiref.compatibility import guarantee_single_callable from opentelemetry import context, trace +from opentelemetry.instrumentation._labeler import ( + enrich_metric_attributes, + get_labeler, + get_labeler_attributes, +) from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, _filter_semconv_active_request_count_attr, @@ -752,6 +797,9 @@ async def __call__( if self.excluded_urls and self.excluded_urls.url_disabled(url): return await self.app(scope, receive, send) + # Required to create new instance for custom attributes in async context + _ = get_labeler() + span_name, additional_attributes = self.default_span_details(scope) attributes = collect_request_attributes( @@ -800,12 +848,14 @@ async def __call__( span_name, scope, receive ) + labeler_metric_attributes = {} otel_send = self._get_otel_send( current_span, span_name, scope, send, attributes, + labeler_metric_attributes, ) await self.app(scope, otel_receive, otel_send) @@ -827,9 +877,21 @@ async def __call__( ) if target: duration_attrs_old[HTTP_TARGET] = target + duration_attrs_old = enrich_metric_attributes( + duration_attrs_old + ) + for key, value in labeler_metric_attributes.items(): + if key not in duration_attrs_old: + duration_attrs_old[key] = value duration_attrs_new = _parse_duration_attrs( attributes, _StabilityMode.HTTP ) + duration_attrs_new = enrich_metric_attributes( + duration_attrs_new + ) + for key, value in labeler_metric_attributes.items(): + if key not in duration_attrs_new: + duration_attrs_new[key] = value span_ctx = set_span_in_context(span) if self.duration_histogram_old: self.duration_histogram_old.record( @@ -979,6 +1041,7 @@ def _get_otel_send( scope, send, duration_attrs, + labeler_metric_attributes, ): expecting_trailers = False @@ -986,6 +1049,9 @@ def _get_otel_send( async def otel_send(message: dict[str, Any]): nonlocal expecting_trailers + if not labeler_metric_attributes: + labeler_metric_attributes.update(get_labeler_attributes()) + status_code = None if message["type"] == "http.response.start": status_code = message["status"] diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py index f716c9f2f1..584e5180f8 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py @@ -22,6 +22,7 @@ import opentelemetry.instrumentation.asgi as otel_asgi from opentelemetry import trace as trace_api +from opentelemetry.instrumentation._labeler import clear_labeler, get_labeler from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, OTEL_SEMCONV_STABILITY_OPT_IN, @@ -272,6 +273,28 @@ async def background_execution_trailers_asgi(scope, receive, send): time.sleep(_SIMULATED_BACKGROUND_TASK_EXECUTION_TIME_S) +async def custom_attrs_asgi(scope, receive, send): + assert isinstance(scope, dict) + assert scope["type"] == "http" + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.method", "POST") + message = await receive() + scope["headers"] = [(b"content-length", b"128")] + if message.get("type") == "http.request": + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [ + [b"Content-Type", b"text/plain"], + [b"content-length", b"1024"], + ], + } + ) + await send({"type": "http.response.body", "body": b"*"}) + + async def error_asgi(scope, receive, send): assert isinstance(scope, dict) assert scope["type"] == "http" @@ -313,6 +336,7 @@ def hook(*_): class TestAsgiApplication(AsyncAsgiTestBase): def setUp(self): super().setUp() + clear_labeler() test_name = "" if hasattr(self, "_testMethodName"): @@ -1553,6 +1577,27 @@ async def test_asgi_metrics_both_semconv(self): self.assertIn(attr, _recommended_attrs_both[metric.name]) self.assertTrue(number_data_point_seen and histogram_data_point_seen) + async def test_asgi_metrics_custom_attributes_skip_override(self): + app = otel_asgi.OpenTelemetryMiddleware(custom_attrs_asgi) + self.seed_app(app) + await self.send_default_request() + await self.get_all_output() + + metrics = self.get_sorted_metrics(SCOPE) + histogram_point_seen = False + for metric in metrics: + if metric.name != "http.server.duration": + continue + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + histogram_point_seen = True + + self.assertTrue(histogram_point_seen) + async def test_asgi_metrics_exemplars_expected_old_semconv(self): """Failing test placeholder asserting exemplars should be present for duration histogram (old semconv).""" app = otel_asgi.OpenTelemetryMiddleware(simple_asgi) diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py index b30423d3bf..97585ac7ef 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py @@ -211,6 +211,54 @@ def response_hook(span: Span, environ: WSGIEnvironment, status: str, response_he To record all of the names set the environment variable ``OTEL_PYTHON_INSTRUMENTATION_HTTP_CAPTURE_ALL_METHODS`` to a value that evaluates to true, e.g. ``1``. +Custom Metrics Attributes using Labeler +*************************************** +The WSGI instrumentation reads from a labeler utility that supports adding custom +attributes to HTTP duration metrics at record time. The custom attributes are +stored only within the context of an instrumented request or operation. The +instrumentor does not overwrite base attributes that exist at the same keys as +any custom attributes. + +.. code-block:: python + + import web + from cheroot import wsgi + + from opentelemetry.instrumentation._labeler import get_labeler + from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware + + urls = ( + '/', 'index', + '/users/(.+)/', 'user_profile' + ) + + class user_profile: + def GET(self, user_id): + # Get the labeler for the current request + labeler = get_labeler() + + # Add custom attributes to WSGI instrumentation metrics + labeler.add("user_id", user_id) + labeler.add("user_type", "registered") + + # Or, add multiple attributes at once + labeler.add_attributes({ + "feature_flag": "new_ui", + "experiment_group": "control" + }) + return f"User profile for {user_id}" + + if __name__ == "__main__": + app = web.application(urls, globals()) + func = app.wsgifunc() + + func = OpenTelemetryMiddleware(func) + + server = wsgi.WSGIServer( + ("localhost", 5100), func, server_name="localhost" + ) + server.start() + API --- """ @@ -223,6 +271,10 @@ def response_hook(span: Span, environ: WSGIEnvironment, status: str, response_he from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, TypeVar, cast from opentelemetry import context, trace +from opentelemetry.instrumentation._labeler import ( + enrich_metric_attributes, + get_labeler_attributes, +) from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, _filter_semconv_active_request_count_attr, @@ -708,6 +760,8 @@ def __call__( response_hook = functools.partial(response_hook, span, environ) start = default_timer() + labeler_metric_attributes = {} + detach_token_in_finally = False self.active_requests_counter.add(1, active_requests_count_attrs) try: with trace.use_span(span): @@ -718,7 +772,12 @@ def __call__( req_attrs, self._sem_conv_opt_in_mode, ) - iterable = self.wsgi(environ, start_response) + try: + iterable = self.wsgi(environ, start_response) + except Exception: + labeler_metric_attributes.update(get_labeler_attributes()) + raise + labeler_metric_attributes.update(get_labeler_attributes()) return _end_span_after_iterating(iterable, span, token) except Exception as ex: if _report_new(self._sem_conv_opt_in_mode): @@ -727,8 +786,7 @@ def __call__( span.set_attribute(ERROR_TYPE, type(ex).__qualname__) span.set_status(Status(StatusCode.ERROR, str(ex))) span.end() - if token is not None: - context.detach(token) + detach_token_in_finally = True raise finally: duration_s = default_timer() - start @@ -737,6 +795,12 @@ def __call__( duration_attrs_old = _parse_duration_attrs( req_attrs, _StabilityMode.DEFAULT ) + duration_attrs_old = enrich_metric_attributes( + duration_attrs_old + ) + for key, value in labeler_metric_attributes.items(): + if key not in duration_attrs_old: + duration_attrs_old[key] = value self.duration_histogram_old.record( max(round(duration_s * 1000), 0), duration_attrs_old, @@ -746,12 +810,20 @@ def __call__( duration_attrs_new = _parse_duration_attrs( req_attrs, _StabilityMode.HTTP ) + duration_attrs_new = enrich_metric_attributes( + duration_attrs_new + ) + for key, value in labeler_metric_attributes.items(): + if key not in duration_attrs_new: + duration_attrs_new[key] = value self.duration_histogram_new.record( max(duration_s, 0), duration_attrs_new, context=active_metric_ctx, ) self.active_requests_counter.add(-1, active_requests_count_attrs) + if detach_token_in_finally and token is not None: + context.detach(token) # Put this in a subfunction to not delay the call to the wrapped diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py index edb6655c5c..8c322dc85d 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py @@ -22,6 +22,7 @@ import opentelemetry.instrumentation.wsgi as otel_wsgi from opentelemetry import trace as trace_api +from opentelemetry.instrumentation._labeler import clear_labeler, get_labeler from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, OTEL_SEMCONV_STABILITY_OPT_IN, @@ -142,6 +143,22 @@ def error_wsgi_unhandled(environ, start_response): raise ValueError +def error_wsgi_unhandled_custom_attrs(environ, start_response): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add_attributes({"endpoint_type": "test", "feature_flag": True}) + assert isinstance(environ, dict) + raise ValueError + + +def error_wsgi_unhandled_override_attrs(environ, start_response): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.method", "POST") + assert isinstance(environ, dict) + raise ValueError + + def wsgi_with_custom_response_headers(environ, start_response): assert isinstance(environ, dict) start_response( @@ -210,6 +227,7 @@ def wsgi_with_repeat_custom_response_headers(environ, start_response): class TestWsgiApplication(WsgiTestBase): def setUp(self): super().setUp() + clear_labeler() test_name = "" if hasattr(self, "_testMethodName"): @@ -451,6 +469,29 @@ def test_wsgi_metrics(self): ) self.assertTrue(number_data_point_seen and histogram_data_point_seen) + def test_wsgi_metrics_custom_attributes_skip_override_old_semconv(self): + app = otel_wsgi.OpenTelemetryMiddleware( + error_wsgi_unhandled_override_attrs + ) + self.assertRaises(ValueError, app, self.environ, self.start_response) + + metrics = self.get_sorted_metrics(SCOPE) + histogram_point_seen = False + + for metric in metrics: + if metric.name != "http.server.duration": + continue + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + histogram_point_seen = True + + self.assertTrue(histogram_point_seen) + def test_wsgi_metrics_exemplars_expected_old_semconv(self): # type: ignore[func-returns-value] """Failing test asserting exemplars should be present for duration histogram (old semconv).""" app = otel_wsgi.OpenTelemetryMiddleware(simple_wsgi) From 875f5ad48030ca36eb018cd41cb834e5f50802c4 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Mar 2026 18:08:25 -0800 Subject: [PATCH 05/12] lint --- .../src/opentelemetry/instrumentation/wsgi/__init__.py | 2 ++ .../tests/test_wsgi_middleware.py | 1 + 2 files changed, 3 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py index 97585ac7ef..dfe5756038 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py @@ -720,6 +720,8 @@ def _start_response( # pylint: disable=too-many-branches # pylint: disable=too-many-locals + # pylint: disable=too-many-public-methods + # pylint: disable=too-many-statements def __call__( self, environ: WSGIEnvironment, start_response: StartResponse ): diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py index 8c322dc85d..eccefe4528 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py @@ -224,6 +224,7 @@ def wsgi_with_repeat_custom_response_headers(environ, start_response): SCOPE = "opentelemetry.instrumentation.wsgi" +# pylint: disable=too-many-public-methods class TestWsgiApplication(WsgiTestBase): def setUp(self): super().setUp() From 8f0e20a2840ad5cb2fabf22192cb2f738ee87b6e Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Mar 2026 18:19:26 -0800 Subject: [PATCH 06/12] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e53d62c877..52bad9bf36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation`: Add experimental metrics attributes Labeler utility ([#4288](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4288)) +- ``opentelemetry-instrumentation-wsgi`, `opentelemetry-instrumentation-asgi`: `enrich_metric_attributes` with any Labeler-stored attributes in Context for HTTP server duration metrics + ([#4300](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4300)) ## Version 1.40.0/0.61b0 (2026-03-04) From a1df5860967c08f1269dd3025d0d66375ea8f726 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 13:35:17 -0800 Subject: [PATCH 07/12] WSGI enrich active_requests counter with Labeler attrs --- .../instrumentation/wsgi/__init__.py | 3 + .../tests/test_wsgi_middleware.py | 109 ++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py index dfe5756038..8090d86f75 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py @@ -738,6 +738,9 @@ def __call__( req_attrs, self._sem_conv_opt_in_mode, ) + active_requests_count_attrs = enrich_metric_attributes( + active_requests_count_attrs + ) span, token = _start_internal_or_server_span( tracer=self.tracer, diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py index eccefe4528..c8b3c53f91 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py @@ -159,6 +159,14 @@ def error_wsgi_unhandled_override_attrs(environ, start_response): raise ValueError +def error_wsgi_unhandled_override_attrs_new_semconv(environ, start_response): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.request.method", "POST") + assert isinstance(environ, dict) + raise ValueError + + def wsgi_with_custom_response_headers(environ, start_response): assert isinstance(environ, dict) start_response( @@ -471,6 +479,82 @@ def test_wsgi_metrics(self): self.assertTrue(number_data_point_seen and histogram_data_point_seen) def test_wsgi_metrics_custom_attributes_skip_override_old_semconv(self): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.method", "POST") + + app = otel_wsgi.OpenTelemetryMiddleware(error_wsgi_unhandled) + self.assertRaises(ValueError, app, self.environ, self.start_response) + + metrics = self.get_sorted_metrics(SCOPE) + active_requests_point_seen = False + histogram_point_seen = False + + for metric in metrics: + if metric.name == "http.server.active_requests": + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, NumberDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + active_requests_point_seen = True + continue + + if metric.name != "http.server.duration": + continue + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertNotIn("custom_attr", point.attributes) + histogram_point_seen = True + + self.assertTrue(active_requests_point_seen) + self.assertTrue(histogram_point_seen) + + def test_wsgi_metrics_custom_attributes_skip_override_new_semconv(self): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.request.method", "POST") + + app = otel_wsgi.OpenTelemetryMiddleware(error_wsgi_unhandled) + self.assertRaises(ValueError, app, self.environ, self.start_response) + + metrics = self.get_sorted_metrics(SCOPE) + active_requests_point_seen = False + histogram_point_seen = False + + for metric in metrics: + if metric.name == "http.server.active_requests": + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, NumberDataPoint) + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + active_requests_point_seen = True + continue + + if metric.name != "http.server.request.duration": + continue + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + self.assertNotIn("custom_attr", point.attributes) + histogram_point_seen = True + + self.assertTrue(active_requests_point_seen) + self.assertTrue(histogram_point_seen) + + def test_wsgi_duration_metrics_custom_attributes_skip_override_old_semconv( + self, + ): app = otel_wsgi.OpenTelemetryMiddleware( error_wsgi_unhandled_override_attrs ) @@ -493,6 +577,31 @@ def test_wsgi_metrics_custom_attributes_skip_override_old_semconv(self): self.assertTrue(histogram_point_seen) + def test_wsgi_duration_metrics_custom_attributes_skip_override_new_semconv( + self, + ): + app = otel_wsgi.OpenTelemetryMiddleware( + error_wsgi_unhandled_override_attrs_new_semconv + ) + self.assertRaises(ValueError, app, self.environ, self.start_response) + + metrics = self.get_sorted_metrics(SCOPE) + histogram_point_seen = False + + for metric in metrics: + if metric.name != "http.server.request.duration": + continue + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + histogram_point_seen = True + + self.assertTrue(histogram_point_seen) + def test_wsgi_metrics_exemplars_expected_old_semconv(self): # type: ignore[func-returns-value] """Failing test asserting exemplars should be present for duration histogram (old semconv).""" app = otel_wsgi.OpenTelemetryMiddleware(simple_wsgi) From a2101ad30f3b254d8863bb274ec7b9ac5b3fc55f Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 13:47:54 -0800 Subject: [PATCH 08/12] ASGI enrich all counter,histo with Labeler attrs --- .../instrumentation/asgi/__init__.py | 3 + .../tests/test_asgi_middleware.py | 138 +++++++++++++++++- 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py index a022ceb5b2..755f245569 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py @@ -818,6 +818,9 @@ async def __call__( attributes, self._sem_conv_opt_in_mode, ) + active_requests_count_attrs = enrich_metric_attributes( + active_requests_count_attrs + ) if scope["type"] == "http": self.active_requests_counter.add(1, active_requests_count_attrs) diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py index 584e5180f8..012d5a9878 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py @@ -295,6 +295,28 @@ async def custom_attrs_asgi(scope, receive, send): await send({"type": "http.response.body", "body": b"*"}) +async def custom_attrs_asgi_new_semconv(scope, receive, send): + assert isinstance(scope, dict) + assert scope["type"] == "http" + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.request.method", "POST") + message = await receive() + scope["headers"] = [(b"content-length", b"128")] + if message.get("type") == "http.request": + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [ + [b"Content-Type", b"text/plain"], + [b"content-length", b"1024"], + ], + } + ) + await send({"type": "http.response.body", "body": b"*"}) + + async def error_asgi(scope, receive, send): assert isinstance(scope, dict) assert scope["type"] == "http" @@ -1584,9 +1606,25 @@ async def test_asgi_metrics_custom_attributes_skip_override(self): await self.get_all_output() metrics = self.get_sorted_metrics(SCOPE) - histogram_point_seen = False + enriched_histogram_metric_names = { + "http.server.duration", + "http.server.response.size", + "http.server.request.size", + } + active_requests_point_seen = False + enriched_histograms_seen = set() for metric in metrics: - if metric.name != "http.server.duration": + if metric.name == "http.server.active_requests": + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, NumberDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertNotIn("custom_attr", point.attributes) + active_requests_point_seen = True + continue + + if metric.name not in enriched_histogram_metric_names: continue data_points = list(metric.data.data_points) self.assertEqual(len(data_points), 1) @@ -1594,9 +1632,101 @@ async def test_asgi_metrics_custom_attributes_skip_override(self): self.assertIsInstance(point, HistogramDataPoint) self.assertEqual(point.attributes[HTTP_METHOD], "GET") self.assertEqual(point.attributes["custom_attr"], "test_value") - histogram_point_seen = True + enriched_histograms_seen.add(metric.name) + + self.assertTrue(active_requests_point_seen) + self.assertSetEqual( + enriched_histogram_metric_names, + enriched_histograms_seen, + ) - self.assertTrue(histogram_point_seen) + async def test_asgi_metrics_custom_attributes_skip_override_new_semconv( + self, + ): + app = otel_asgi.OpenTelemetryMiddleware(custom_attrs_asgi_new_semconv) + self.seed_app(app) + await self.send_default_request() + await self.get_all_output() + + metrics = self.get_sorted_metrics(SCOPE) + enriched_histogram_metric_names = { + "http.server.request.duration", + "http.server.response.body.size", + "http.server.request.body.size", + } + active_requests_point_seen = False + enriched_histograms_seen = set() + for metric in metrics: + if metric.name == "http.server.active_requests": + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, NumberDataPoint) + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + self.assertNotIn("custom_attr", point.attributes) + active_requests_point_seen = True + continue + + if metric.name not in enriched_histogram_metric_names: + continue + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + enriched_histograms_seen.add(metric.name) + + self.assertTrue(active_requests_point_seen) + self.assertSetEqual( + enriched_histogram_metric_names, + enriched_histograms_seen, + ) + + async def test_asgi_active_requests_attrs_use_enrich_old_semconv(self): + with mock.patch( + "opentelemetry.instrumentation.asgi.enrich_metric_attributes", + wraps=otel_asgi.enrich_metric_attributes, + ) as mock_enrich: + app = otel_asgi.OpenTelemetryMiddleware(simple_asgi) + self.seed_app(app) + await self.send_default_request() + await self.get_all_output() + + enriched_active_attrs_seen = False + for call in mock_enrich.call_args_list: + if not call.args: + continue + attrs = call.args[0] + if HTTP_METHOD in attrs and HTTP_STATUS_CODE not in attrs: + enriched_active_attrs_seen = True + break + + self.assertTrue(enriched_active_attrs_seen) + + async def test_asgi_active_requests_attrs_use_enrich_new_semconv(self): + with mock.patch( + "opentelemetry.instrumentation.asgi.enrich_metric_attributes", + wraps=otel_asgi.enrich_metric_attributes, + ) as mock_enrich: + app = otel_asgi.OpenTelemetryMiddleware(simple_asgi) + self.seed_app(app) + await self.send_default_request() + await self.get_all_output() + + enriched_active_attrs_seen = False + for call in mock_enrich.call_args_list: + if not call.args: + continue + attrs = call.args[0] + if ( + HTTP_REQUEST_METHOD in attrs + and HTTP_RESPONSE_STATUS_CODE not in attrs + ): + enriched_active_attrs_seen = True + break + + self.assertTrue(enriched_active_attrs_seen) async def test_asgi_metrics_exemplars_expected_old_semconv(self): """Failing test placeholder asserting exemplars should be present for duration histogram (old semconv).""" From e76c98cbb7b9705f04db0f8dcbbb8abb8b5b4064 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 14:07:06 -0800 Subject: [PATCH 09/12] Docstring --- .../instrumentation/asgi/__init__.py | 18 +++++++++++++----- .../instrumentation/wsgi/__init__.py | 14 +++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py index 755f245569..b93edad964 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py @@ -206,11 +206,19 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]): Custom Metrics Attributes using Labeler *************************************** -The ASGI instrumentation reads from a labeler utility that supports adding custom -attributes to HTTP duration metrics at record time. The custom attributes are -stored only within the context of an instrumented request or operation. The -instrumentor does not overwrite base attributes that exist at the same keys as -any custom attributes. +The ASGI instrumentation reads custom attributes from the labeler (when present) +and applies them to all HTTP server metric points emitted by the middleware: + +- Active requests counter (`http.server.active_requests`) +- Duration histograms (`http.server.duration` and/or + `http.server.request.duration` depending on semantic convention mode) +- Response size histograms (`http.server.response.size` and/or + `http.server.response.body.size`) +- Request size histograms (`http.server.request.size` and/or + `http.server.request.body.size`) + +Labeler attributes are request-scoped and merged without overriding base metric +attributes at the same keys. .. code-block:: python diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py index 8090d86f75..8d8d6c35d5 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py @@ -213,11 +213,15 @@ def response_hook(span: Span, environ: WSGIEnvironment, status: str, response_he Custom Metrics Attributes using Labeler *************************************** -The WSGI instrumentation reads from a labeler utility that supports adding custom -attributes to HTTP duration metrics at record time. The custom attributes are -stored only within the context of an instrumented request or operation. The -instrumentor does not overwrite base attributes that exist at the same keys as -any custom attributes. +The WSGI instrumentation reads custom attributes from the labeler (when present) +and applies them to all HTTP server metric points emitted by the middleware: + +- Active requests counter (`http.server.active_requests`) +- Duration histograms (`http.server.duration` and/or + `http.server.request.duration` depending on semantic convention mode) + +Labeler attributes are request-scoped and merged without overriding base metric +attributes at the same keys. .. code-block:: python From 23a9dac90183e83b0cc98691b4e10a5399f45df3 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 15:04:52 -0800 Subject: [PATCH 10/12] Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52bad9bf36..d0cad9e6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation`: Add experimental metrics attributes Labeler utility ([#4288](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4288)) -- ``opentelemetry-instrumentation-wsgi`, `opentelemetry-instrumentation-asgi`: `enrich_metric_attributes` with any Labeler-stored attributes in Context for HTTP server duration metrics +- `opentelemetry-instrumentation-wsgi`, `opentelemetry-instrumentation-asgi`: `enrich_metric_attributes` with any Labeler-stored attributes in Context ([#4300](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4300)) ## Version 1.40.0/0.61b0 (2026-03-04) From 9cdf6631bbd68d72f7570756c3c23f0f3f5a5ffb Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 15:10:35 -0800 Subject: [PATCH 11/12] Falcon enrich_metric_attributes from Labeler if set --- .../instrumentation/falcon/__init__.py | 54 ++++++++++++++++++- .../tests/app.py | 17 ++++++ .../tests/test_falcon.py | 52 ++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py index 09429f8bb5..2e6b12fc57 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py @@ -180,6 +180,49 @@ def response_hook(span, req, resp): Note: The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. +Custom Metrics Attributes using Labeler +*************************************** +The Falcon instrumentation reads from a labeler utility that supports adding custom +attributes to HTTP server metrics at emit time, including: + +- Active requests counter (``http.server.active_requests``) +- Duration histogram (``http.server.duration``) +- Request duration histogram (``http.server.request.duration``) + +The custom attributes are stored only within the context of an instrumented +request or operation. The instrumentor does not overwrite base attributes that +exist at the same keys as any custom attributes. + + +.. code-block:: python + + import falcon + + from opentelemetry.instrumentation._labeler import get_labeler + from opentelemetry.instrumentation.falcon import FalconInstrumentor + + FalconInstrumentor().instrument() + app = falcon.App() + + class UserProfileResource: + def on_get(self, req, resp, user_id): + # Get the labeler for the current request + labeler = get_labeler() + + # Add custom attributes to Falcon instrumentation metrics + labeler.add("user_id", user_id) + labeler.add("user_type", "registered") + + # Or, add multiple attributes at once + labeler.add_attributes({ + "feature_flag": "new_ui", + "experiment_group": "control" + }) + + resp.text = f'User profile for {user_id}' + + app.add_route('/users/{user_id}/', UserProfileResource()) + API --- """ @@ -195,6 +238,7 @@ def response_hook(span, req, resp): import opentelemetry.instrumentation.wsgi as otel_wsgi from opentelemetry import context, trace +from opentelemetry.instrumentation._labeler import enrich_metric_attributes from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, _get_schema_url, @@ -373,7 +417,9 @@ def __call__(self, env, start_response): attributes, self._sem_conv_opt_in_mode ) ) - self.active_requests_counter.add(1, active_requests_count_attrs) + self.active_requests_counter.add( + 1, enrich_metric_attributes(active_requests_count_attrs) + ) if span.is_recording(): if span.is_recording() and span.kind == trace.SpanKind.SERVER: @@ -408,6 +454,7 @@ def _start_response(status, response_headers, *args, **kwargs): duration_attrs = otel_wsgi._parse_duration_attrs( attributes, _StabilityMode.DEFAULT ) + duration_attrs = enrich_metric_attributes(duration_attrs) self.duration_histogram_old.record( max(round(duration_s * 1000), 0), duration_attrs ) @@ -415,11 +462,14 @@ def _start_response(status, response_headers, *args, **kwargs): duration_attrs = otel_wsgi._parse_duration_attrs( attributes, _StabilityMode.HTTP ) + duration_attrs = enrich_metric_attributes(duration_attrs) self.duration_histogram_new.record( max(duration_s, 0), duration_attrs ) - self.active_requests_counter.add(-1, active_requests_count_attrs) + self.active_requests_counter.add( + -1, enrich_metric_attributes(active_requests_count_attrs) + ) if exception is None: activation.__exit__(None, None, None) else: diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py index 416ac80dff..fc6d704ce6 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py @@ -1,6 +1,8 @@ import falcon from packaging import version as package_version +from opentelemetry.instrumentation._labeler import get_labeler + # pylint:disable=R0201,W0613,E0602 @@ -75,6 +77,20 @@ def on_get(self, req, resp, user_id): resp.text = f"Hello user {user_id}" +class UserLabelerResource: + def on_get(self, req, resp, user_id): + labeler = get_labeler() + labeler.add("custom_attr", "test_value") + labeler.add("http.method", "POST") + # pylint: disable=no-member + resp.status = falcon.HTTP_200 + + if _parsed_falcon_version < package_version.parse("3.0.0"): + resp.body = f"Hello user {user_id}" + else: + resp.text = f"Hello user {user_id}" + + def make_app(): if _parsed_falcon_version < package_version.parse("3.0.0"): # Falcon 1 and Falcon 2 @@ -90,5 +106,6 @@ def make_app(): "/test_custom_response_headers", CustomResponseHeaderResource() ) app.add_route("/user/{user_id}", UserResource()) + app.add_route("/user_custom_attr/{user_id}", UserLabelerResource()) return app diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py index d56a0f1303..ab699f1b8d 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py @@ -21,6 +21,7 @@ from packaging import version as package_version from opentelemetry import trace +from opentelemetry.instrumentation._labeler import clear_labeler from opentelemetry.instrumentation._semconv import ( HTTP_DURATION_HISTOGRAM_BUCKETS_NEW, OTEL_SEMCONV_STABILITY_OPT_IN, @@ -126,6 +127,7 @@ class TestFalconBase(TestBase): def setUp(self): super().setUp() + clear_labeler() test_name = "" if hasattr(self, "_testMethodName"): @@ -540,6 +542,56 @@ def test_falcon_metrics(self): self.assertIn(attr, _recommended_attrs[metric.name]) self.assertTrue(number_data_point_seen and histogram_data_point_seen) + def test_falcon_metrics_custom_attributes_skip_override(self): + self.client().simulate_get("/user_custom_attr/123") + metrics = self.get_sorted_metrics(SCOPE) + active_requests_point_seen = False + histogram_point_seen = False + + for metric in metrics: + if metric.name == "http.server.active_requests": + data_points = list(metric.data.data_points) + for point in data_points: + self.assertIsInstance(point, NumberDataPoint) + if point.attributes.get("custom_attr") != "test_value": + continue + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + active_requests_point_seen = True + continue + + if metric.name != "http.server.duration": + continue + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + point = data_points[0] + self.assertIsInstance(point, HistogramDataPoint) + self.assertEqual(point.attributes[HTTP_METHOD], "GET") + self.assertEqual(point.attributes["custom_attr"], "test_value") + histogram_point_seen = True + + self.assertTrue(active_requests_point_seen) + self.assertTrue(histogram_point_seen) + + def test_falcon_metrics_active_requests_custom_attributes_new_semconv( + self, + ): + self.client().simulate_get("/user_custom_attr/123") + metrics = self.get_sorted_metrics(SCOPE) + active_requests_point_seen = False + + for metric in metrics: + if metric.name != "http.server.active_requests": + continue + data_points = list(metric.data.data_points) + for point in data_points: + self.assertIsInstance(point, NumberDataPoint) + if point.attributes.get("custom_attr") != "test_value": + continue + self.assertEqual(point.attributes[HTTP_REQUEST_METHOD], "GET") + active_requests_point_seen = True + + self.assertTrue(active_requests_point_seen) + def test_falcon_metric_values_new_semconv(self): number_data_point_seen = False histogram_data_point_seen = False From 39e1959fd9d53b053e0eb6e505abee67cb512936 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 Mar 2026 15:12:22 -0800 Subject: [PATCH 12/12] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cad9e6ef..09c241e1e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#4288](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4288)) - `opentelemetry-instrumentation-wsgi`, `opentelemetry-instrumentation-asgi`: `enrich_metric_attributes` with any Labeler-stored attributes in Context ([#4300](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4300)) +- `opentelemetry-instrumentation-falcon`: `enrich_metric_attributes` with any Labeler-stored attributes in Context + ([#4309](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4309)) ## Version 1.40.0/0.61b0 (2026-03-04)