feat: allow HTTP operations to be configurable via environment variable (#718)

*Issue #, if available:*

*Description of changes:*
Same as
aws-observability/aws-otel-java-instrumentation#1352

When HTTP span names don't contain a URL path, we generate the HTTP
operation by truncating the URL path to only the first trailing value to
preserve low cardinality (i.e. /api/v1/users -> /api). This can result
in overly broad operation groupings for services with endpoint paths of
various depths.

This PR introduces an environment variable configuration,
`OTEL_AWS_HTTP_OPERATION_PATHS`, which allows users to configure their
own HTTP endpoint paths. If this variable is provided, the span name's
URL path will resolve to the longest matching path. Wildcards are
supported with the following syntaxes: `{version}`, `:version`, or
simply `*`. This way, users can decide how their service endpoint are
grouped into operation names shown in CloudWatch.

Added unit and integration tests to verify behavior, and did some E2E
testing with an instrumented HTTP server.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: ADOT Patch workflow <adot-patch-workflow@github.com>

Author: ezhang6811

CHANGELOG.md

@@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
 
 ## Unreleased
 
+- feat: support environment-configured endpoint visibility for HTTP operation names
+  ([#718](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/718))
 - fix(lambda-layer): Disable all agentic instrumentation in Lambda by default
   ([#710](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/710))
 - fix(genai-instrumentors): cleanup code, align with OTel GenAI semconv, add missing attributes and fix deprecated usage

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py

@@ -26,6 +26,12 @@
 # Max keyword length supported by parsing into remote_operation from DB_STATEMENT
 MAX_KEYWORD_LENGTH = 27
 
+# Environment variable for configurable operation name paths
+OTEL_AWS_HTTP_OPERATION_PATHS_CONFIG: str = "OTEL_AWS_HTTP_OPERATION_PATHS"
+
+# Cached parsed operation paths (sorted longest first)
+_operation_paths: List[str] = None
+
 
 # Get dialect keywords retrieved from dialect_keywords.json file.
 # Only meant to be invoked by SQL_KEYWORD_PATTERN and unit tests
@@ -41,6 +47,112 @@ def _get_dialect_keywords() -> List[str]:
 SQL_KEYWORD_PATTERN = r"^(?:" + "|".join(_get_dialect_keywords()) + r")\b"
 
 
+def get_operation_paths() -> List[str]:
+    """Parse OTEL_AWS_HTTP_OPERATION_PATHS env var into a sorted list of path templates (longest first)."""
+    global _operation_paths  # pylint: disable=global-statement
+    if _operation_paths is None:
+        config = os.environ.get(OTEL_AWS_HTTP_OPERATION_PATHS_CONFIG)
+        if config is None or config.strip() == "":
+            _operation_paths = []
+        else:
+            paths = [p.strip() for p in config.split(",") if p.strip()]
+            # Sort longest first (by segment count) for longest-prefix-match
+            paths.sort(key=lambda p: len(p.split("/")), reverse=True)
+            _operation_paths = paths
+    return _operation_paths
+
+
+def reset_operation_paths() -> None:
+    """Reset cached operation paths (for testing)."""
+    global _operation_paths  # pylint: disable=global-statement
+    _operation_paths = None
+
+
+def apply_operation_path_span_name(span: ReadableSpan) -> ReadableSpan:
+    """If OTEL_AWS_HTTP_OPERATION_PATHS is configured and matches, override the span name.
+
+    Returns the span (possibly with modified _name) if a match is found, or the original span unchanged.
+    """
+    paths = get_operation_paths()
+    if not paths:
+        return span
+
+    url_path = _get_url_path(span)
+    if url_path is None or url_path == "":
+        return span
+
+    # Strip query string and fragment (relevant for http.target)
+    for sep in ("?", "#"):
+        idx = url_path.find(sep)
+        if idx >= 0:
+            url_path = url_path[:idx]
+
+    # Normalize trailing slashes
+    while url_path.endswith("/") and len(url_path) > 1:
+        url_path = url_path[:-1]
+
+    url_segments = url_path.split("/")
+    for pattern in paths:
+        normalized_pattern = pattern
+        while normalized_pattern.endswith("/") and len(normalized_pattern) > 1:
+            normalized_pattern = normalized_pattern[:-1]
+        if _segments_match(url_segments, normalized_pattern.split("/")):
+            http_method = _get_http_method(span)
+            new_name = f"{http_method} {pattern}" if http_method else pattern
+            # Override the span name directly
+            span._name = new_name
+            return span
+
+    return span
+
+
+def _get_url_path(span: ReadableSpan) -> str:
+    """Get the URL path from the span, checking url.path first, then falling back to http.target (deprecated)."""
+    if span.attributes is None:
+        return None
+    url_path = span.attributes.get(SpanAttributes.URL_PATH)
+    if url_path is not None:
+        return url_path
+    return span.attributes.get(SpanAttributes.HTTP_TARGET)
+
+
+def _get_http_method(span: ReadableSpan) -> str:
+    """Get the HTTP method from the span, checking http.request.method first, then http.method (deprecated)."""
+    if span.attributes is None:
+        return None
+    method = span.attributes.get(SpanAttributes.HTTP_REQUEST_METHOD)
+    if method is not None:
+        return method
+    return span.attributes.get(SpanAttributes.HTTP_METHOD)
+
+
+def _segments_match(url_segments: List[str], pattern_segments: List[str]) -> bool:
+    """Check if URL segments match a pattern's segments.
+
+    Pattern segments can use {param}, :param, or * as wildcards matching any single non-empty segment.
+    The pattern acts as a prefix — extra URL segments after the pattern are allowed.
+    """
+    for idx, ps in enumerate(pattern_segments):
+        if idx >= len(url_segments):
+            return False
+        us = url_segments[idx]
+
+        if _is_wildcard_segment(ps):
+            if us == "":
+                return False
+            continue
+
+        if ps != us:
+            return False
+
+    return True
+
+
+def _is_wildcard_segment(segment: str) -> bool:
+    """A segment is a wildcard if it uses {param}, :param, or * format."""
+    return (segment.startswith("{") and segment.endswith("}")) or segment.startswith(":") or segment == "*"
+
+
 def get_ingress_operation(__, span: ReadableSpan) -> str:
     """
     Ingress operation (i.e. operation for Server and Consumer spans) will be generated from "http.method + http.target/
@@ -166,8 +278,8 @@ def _is_valid_operation(span: ReadableSpan, operation: str) -> bool:
     if operation is None or operation == UNKNOWN_OPERATION:
         return False
 
-    if is_key_present(span, SpanAttributes.HTTP_METHOD):
-        http_method: str = span.attributes.get(SpanAttributes.HTTP_METHOD)
+    http_method: str = _get_http_method(span)
+    if http_method:
         return operation != http_method
 
     return True

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_span_metrics_processor.py

@@ -5,6 +5,7 @@
 from typing_extensions import override
 
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_REMOTE_SERVICE
+from amazon.opentelemetry.distro._aws_span_processing_util import apply_operation_path_span_name
 from amazon.opentelemetry.distro.metric_attribute_generator import MetricAttributeGenerator
 from amazon.opentelemetry.distro.sampler.aws_xray_remote_sampler import AwsXRayRemoteSampler
 from opentelemetry.context import Context
@@ -86,6 +87,10 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None
 
     @override
     def on_end(self, span: ReadableSpan) -> None:
+        # If OTEL_AWS_HTTP_OPERATION_PATHS is configured, override the span name
+        # so that metrics use the configured operation path instead of the original span name.
+        span = apply_operation_path_span_name(span)
+
         attribute_dict: Dict[str, BoundedAttributes] = self._generator.generate_metric_attributes_dict_from_span(
             span, self._resource
         )

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_span_processing_util.py

@@ -381,3 +381,199 @@ def test_sql_dialect_keywords_max_length(self):
         keywords: List[str] = _get_dialect_keywords()
         for keyword in keywords:
             self.assertLessEqual(len(keyword), MAX_KEYWORD_LENGTH)
+
+
+class TestOperationPaths(TestCase):
+    """Tests for OTEL_AWS_HTTP_OPERATION_PATHS and apply_operation_path_span_name."""
+
+    def setUp(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import reset_operation_paths
+
+        reset_operation_paths()
+        self.span_mock = MagicMock()
+        self.span_mock.attributes = {}
+        self.span_mock.name = "GET /api"
+        self.span_mock._name = "GET /api"
+
+    def tearDown(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import reset_operation_paths
+
+        reset_operation_paths()
+
+    # --- _segments_match tests ---
+
+    def test_segments_match_exact(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertTrue(_segments_match("/api/contests".split("/"), "/api/contests".split("/")))
+
+    def test_segments_match_no_match(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertFalse(_segments_match("/api/players".split("/"), "/api/contests".split("/")))
+
+    def test_segments_match_extra_url_segments_allowed(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertTrue(_segments_match("/api/contests/123/extra".split("/"), "/api/contests".split("/")))
+
+    def test_segments_match_pattern_longer_than_url(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertFalse(_segments_match("/api".split("/"), "/api/contests/{id}".split("/")))
+
+    def test_segments_match_curly_brace_wildcard(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertTrue(_segments_match("/api/contests/123".split("/"), "/api/contests/{id}".split("/")))
+
+    def test_segments_match_colon_param_wildcard(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertTrue(_segments_match("/api/users/42".split("/"), "/api/users/:userId".split("/")))
+
+    def test_segments_match_star_wildcard(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertTrue(
+            _segments_match("/api/contests/123/leaderboard".split("/"), "/api/contests/*/leaderboard".split("/"))
+        )
+
+    def test_segments_match_wildcard_does_not_match_empty(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import _segments_match
+
+        self.assertFalse(_segments_match("/api/contests/".split("/"), "/api/contests/{id}".split("/")))
+
+    # --- apply_operation_path_span_name tests ---
+
+    @patch.dict(
+        os.environ,
+        {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests/{id}/leaderboard, /api/contests/{id}, /api/contests"},
+    )
+    def test_apply_matches_url_path(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/contests/123/leaderboard", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/contests/{id}/leaderboard")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/teams/{id}, /api/teams"})
+    def test_apply_matches_http_target_with_query(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"http.target": "/api/teams/5?include=roster", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/teams/{id}")
+
+    @patch.dict(
+        os.environ,
+        {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests/{id}/leaderboard, /api/contests/{id}, /api/contests, /api"},
+    )
+    def test_apply_longest_match_wins(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/contests/42", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/contests/{id}")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests/{id}"})
+    def test_apply_no_match_returns_original(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/unknown/path", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api")  # unchanged
+
+    def test_apply_empty_config_returns_original(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import apply_operation_path_span_name
+
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertIs(result, self.span_mock)
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests"})
+    def test_apply_no_http_method(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/contests"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "/api/contests")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/v1/{userId}, /api/{version}/user1"})
+    def test_apply_same_length_first_config_wins(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/v1/user1", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/v1/{userId}")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests"})
+    def test_apply_trailing_slash_normalized(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/contests/", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/contests")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/contests"})
+    def test_apply_query_string_stripped(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/contests?page=1&size=10", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/contests")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/users/:userId/stats"})
+    def test_apply_colon_param_in_config(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/users/42/stats", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/users/:userId/stats")
+
+    @patch.dict(os.environ, {"OTEL_AWS_HTTP_OPERATION_PATHS": "/api/*/users/*"})
+    def test_apply_star_wildcard_in_config(self):
+        from amazon.opentelemetry.distro._aws_span_processing_util import (
+            apply_operation_path_span_name,
+            reset_operation_paths,
+        )
+
+        reset_operation_paths()
+        self.span_mock.attributes = {"url.path": "/api/v2/users/42", "http.request.method": "GET"}
+        result = apply_operation_path_span_name(self.span_mock)
+        self.assertEqual(result._name, "GET /api/*/users/*")