Skip to content
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
959e752
feat: Add OpenTelemetry environment variable and options configuratio…
chalmerlowe Jun 22, 2026
071ae3c
feat(observability): add base OpenTelemetry span enricher interceptor
chalmerlowe Jun 22, 2026
626ed58
test(observability): add test-only environment variable overrides and…
chalmerlowe Jun 22, 2026
c867011
feat(observability): simplify options resolver to tracing-only
chalmerlowe Jun 24, 2026
3f6ba78
feat(observability): remove OtelSpanEnricher from current PR
chalmerlowe Jun 24, 2026
35f0736
test(o11y): add coverage for environment overrides and fallback
chalmerlowe Jun 24, 2026
fa5d234
test(o11y): improve test isolation in environment overrides
chalmerlowe Jun 24, 2026
c19700e
test(o11y): eliminate 'traces' references and improve test readability
chalmerlowe Jul 6, 2026
5aedd86
refactor(o11y): add license header, reorganize variables and improve …
chalmerlowe Jul 6, 2026
4e49506
chore(o11y): apply black formatting fixes to tests
chalmerlowe Jul 6, 2026
4e950c1
feat(otel): align resolve_feature_flags with strict LLD requirements
chalmerlowe Jul 14, 2026
c6fe8a6
test(o11y): add comprehensive tests for generic feature resolver
chalmerlowe Jul 15, 2026
391fe4c
feat(o11y): refactor resolve_feature_flags to be generic and strictly…
chalmerlowe Jul 15, 2026
0f92c35
style(o11y): apply black formatting to options.py and test_options.py
chalmerlowe Jul 15, 2026
2fb0057
docs(o11y): revise terminology to remove 'gate' metaphor
chalmerlowe Jul 16, 2026
57014ab
fix(o11y): add warning for malformed environment variables
chalmerlowe Jul 16, 2026
1c17863
refactor(o11y): make _has_provider more robust by using getattr
chalmerlowe Jul 16, 2026
8dd6175
test(o11y): add missing tests for experimental path without provider
chalmerlowe Jul 16, 2026
9fc0088
style(o11y): apply black formatting to test_options.py
chalmerlowe Jul 16, 2026
11313ec
feat: move observability resolver to general purpose feature_gating_h…
chalmerlowe Jul 22, 2026
9b4df5c
refactor: remove custom test overrides and use monkeypatch
chalmerlowe Jul 22, 2026
745cceb
chore: remove opentelemetry-api dependency from google-api-core
chalmerlowe Jul 22, 2026
d8d313b
style: apply black formatting to test_feature_gating_helpers.py
chalmerlowe Jul 22, 2026
71330d9
refactor: rename parameters and enforce keyword-only in feature gating
chalmerlowe Jul 23, 2026
bd6d88f
feat: add dunder key guardrail to feature gating
chalmerlowe Jul 23, 2026
f667db5
refactor: update tests to use private module name
chalmerlowe Jul 23, 2026
c72bc30
refactor: modernize type hints in feature gating helpers
chalmerlowe Jul 23, 2026
bff8b6a
feat: use custom FeatureGatingError in feature gating helpers
chalmerlowe Jul 23, 2026
526c0b8
refactor: rename _has_provider to _has_feature_key for better general…
chalmerlowe Jul 23, 2026
050a857
docs: scrub remaining 'provider' terminology from helper module
chalmerlowe Jul 23, 2026
d9aff39
chore: apply ruff formatting to feature gating helpers
chalmerlowe Jul 23, 2026
23b87c0
Update packages/google-api-core/testing/constraints-3.10.txt
chalmerlowe Jul 23, 2026
d8065b1
Update packages/google-api-core/testing/constraints-async-rest-3.10.txt
chalmerlowe Jul 23, 2026
1e98bde
Apply suggestion from @chalmerlowe
chalmerlowe Jul 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions packages/google-api-core/google/api_core/feature_gating_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
# -*- coding: utf-8 -*-
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Observability environment variable and client options resolution helpers."""

import os
import warnings
from typing import Any, Dict, Optional, Union

# Allowed truthy and falsy patterns for environment variables
_TRUTHY_VALUES = ("y", "yes", "t", "true", "on", "1")
_FALSY_VALUES = ("n", "no", "f", "false", "off", "0")


def _strtobool(val: str) -> Optional[bool]:
"""Convert a string representation of truth to a boolean."""
clean_val = val.lower().strip()
if not clean_val:
return None
if clean_val in _TRUTHY_VALUES:
return True
if clean_val in _FALSY_VALUES:
return False
raise ValueError(f"Invalid truth value: {val!r}")


def _get_env_bool(name: str) -> Optional[bool]:
"""Retrieve the boolean value of an environment variable."""
val = os.getenv(name)
if val is None:
return None
try:
return _strtobool(val)
except ValueError as e:
warnings.warn(f"Ignored invalid value for {name}: {e}", RuntimeWarning)
return None


def _has_provider(
client_options: Optional[Union[Dict[str, Any], Any]], provider_key: str
) -> bool:
"""Checks if a specific provider key is present and not None in client_options."""
if client_options is None:
return False

if isinstance(client_options, dict):
return client_options.get(provider_key) is not None

return getattr(client_options, provider_key, None) is not None

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should have some limitations provider_key? Like returning False for __?

Something like __class__ or __dict__ would return True, but probably not what we're looking for

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We added protections against reading dunder_methods.



def resolve_feature_flags(
env_var: str,
provider_key: str,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I wonder if the provider naming is general enough? "Provider" is the term used for Otel, but does it apply equally to all features? Maybe this should be called feature_key? Or flag_id?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We updated the arg name.

client_options: Optional[Union[Dict[str, Any], Any]] = None,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume Any is for ClientOptions? Why not use the class directly?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Redundant

This hint is a bit redundant. Since Any covers everything (including dictionaries), putting Dict[str, Any] inside the Union with Any doesn't restrict anything for a type checker.

Optional[Union[Dict[str, Any], Any]]

Clarity

If we change it to:

client_options: Optional[Union[Dict[str, Any], ClientOptions]] = None

We do get some clarity. An engineer looking at the function signature knows exactly what is expected in one scenario.

Duck Typing

However, there is a reason to prefer Any (or object) here. It gives our customers flexibility to use their own custom options classes, as long as they implement the interface our code expects (i.e they can use duck typing).

In our implementation of _has_provider:

    if isinstance(client_options, dict):
        return client_options.get(provider_key) is not None
    return getattr(client_options, provider_key, None) is not None

We use getattr(). This means our helper doesn't care if the object is strictly a ClientOptions. It can be any object that happens to have the attribute we are looking for. We can see this intent in our tests in test_feature_gating_helpers.py:

class _MockOptions:
    def __init__(self):
        self.other_option = "value"

We pass this _MockOptions object to resolve_feature_flags. It is NOT a subclass of ClientOptions.

If we change the type hint to ClientOptions, strict type checkers (like mypy) will claim that _MockOptions is not a valid argument.
If we keep it as Any (or object), the type checker stays quiet because it understands we are using duck typing.

@daniel-sanche daniel-sanche Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How common do you expect that use case to be? Will customers even have reason to call this code? And if they did, couldn't they also achieve this by either:

  • subclassing ClientOptions with their own implementation
  • passing in a dict
  • using typing.cast

Personally, I think allowing arbitrary types here feels overkill, and potentially dangerous. Dict[str, Any] | ClientOptions makes more sense to me. Or really, ideally just one or the other

But it sounds like you have plans to extend this in the future, so maybe I'm missing context. If you want to keep it fully permissive, I'd suggest:

  • rename client_options to something more general
  • use object instead of Any (since only objects have getattr)
  • include ClientOptions in the type annotation, for the same reason you include Dict

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We included ClientOptions in the type annotation.

) -> bool:
"""Determines if a feature is enabled based on environment variables and client options.

Behavior depends on whether the `env_var` name contains "EXPERIMENTAL":

- **Experimental Path** (env_var contains "EXPERIMENTAL"):
Strict control. Requires the environment variable to be explicitly 'true'.
If a programmatic provider is passed but the environment variable is not 'true',
raises ValueError (Fail Fast).

- **GA Path** (env_var does not contain "EXPERIMENTAL"):
Standard precedence. Enabled if a programmatic provider is passed,
otherwise falls back to the environment variable value.

Args:
env_var: The name of the environment variable controlling this feature.
provider_key: The key in client_options/attributes for the programmatic provider.
client_options: Optional. A dictionary or object containing client configuration.

Returns:
bool: True if the feature is resolved to enabled, False otherwise.

Raises:
ValueError: If a provider is provided for an experimental feature without enabling the experimental environment variable.
"""

# Check for programmatic feature provider
has_provider = _has_provider(client_options, provider_key)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this helper wouldn't be needed if you can simplify the client_options type (either by disallowing dicts, or doing a one-liner to convert them)

options = ClientOptions.from_dict(client_options) if isinstance(client_options, dict) else client_options
has_provider = getattr(options, provider_key, None) is not None

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer to keep the code in the resolver lean and extensible thus the choice to use helper functions like _has_provider

Especially considering that at a future time I would like to add the ability for the resolver to resolve:

  • legacy environment variables (such as multiple env vars that we already have in Spanner and Storage)

In addition, there is also some benefit to being able to set env vars on a language level (turn on o11y in Python by not Node) OR on a per-service level (turn on o11y in Bigtable but not Firestore).

Being able to slip tiny helper functions into the resolver helps keep the resolution logic and precedence order the main priority AND not "how should we determine if something is set OR not set and how is that different from how we parsed providers 12 lines up OR parsed this customer env var"

  • Language-wide env vars (e.g., GOOGLE_CLOUD_<LANGUAGE>_TRACING_ENABLED)
  • Service-Specific env vars (e.g., GOOGLE_CLOUD_<SERVICE>_TRACING_ENABLED
  • etc

If we are running this function multiple times in a row (e.g., during client initialization to check GOOGLE_SDK_PYTHON_TRACING_ENABLED, then GOOGLE_SDK_PYTHON_METRICS_ENABLED, etc.), we want resolve_feature_flags to be straight forward for troubleshooting and maintenance.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It sounds like the function signature for resolve_feature_flags could be changing in the future then? Do you know what it would look like? We have to be careful about breaking changes, so if there's anything we can do now to make it more flexible, that would be great

I don't completely understand the plan yet, but a couple things to consider:

  • client_options could be given a more general name (like configuration), if we expect other config types in the future
  • We could add * to make client_options keyword only, so we can insert others in any order later
  • env_var and client_options could accept lists instead of single items (although I think we could add this without a breaking change later)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are good ideas for future-proofing. We agree on making arguments keyword-only and using more general names. We updated the signature to make all arguments keyword-only (placing * at the start) to prevent accidental parameter swapping and allow easy expansion later. We have also renamed provider_key to feature_key and client_options to configuration to reflect that this is a generic internal helper.

Regarding lists: the resolver is currently intended to be a simple one-specific-input, one-specific-output (e.g. a boolean) gate, so the idea of passing in lists of configs is not in line with the expected use. If we need to resolve several gates, we expect to call it multiple times, keeping the logic simple:

is_tracing_enabled = resolve_feature_flags(env_var="GOOGLE_CLOUD_PYTHON_TRACING_ENABLED") 
is_metrics_enabled = resolve_feature_flags(env_var="GOOGLE_CLOUD_EXPERIMENTAL_PYTHON_METRICS_ENABLED", feature_key="metrics_provider", configuration=client_options) 
is_creds_feature_enabled = resolve_feature_flags(env_var="GOOGLE_CLOUD_CREDENTIALS_FEATURE_ENABLED", feature_key="credential_type", configuration=credentials)


# Read environment variable
env_var_setting = _get_env_bool(env_var)

# EXPERIMENTAL PATH:
# Resolution Hierarchy:
# 1. EXPERIMENTAL Environment Variable
# 2. Fail Fast if Provider present but EXPERIMENTAL Environment Variable is not enabled
if "EXPERIMENTAL" in env_var:
# Fail Fast if provider present but experimental environment variable is not enabled
if env_var_setting is not True and has_provider:
raise ValueError(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think a custom error would be helpful here? ValueError seems a like an imperfect fit for this kind of thing. And, assuming this is being called as part of client init, we may be rasing other ValueErrors for other reasons, so it might be good to differentiate them some more

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We updated to a custom error.

f"Experimental feature requires {env_var} to be set to 'true' to use programmatic providers."
)

return bool(env_var_setting)

# GENERAL AVAILABILITY PATH:
# Resolution Hierarchy:
# 1. Programmatic Provider
# 2. Environment Variable

# Check Programmatic Provider
if has_provider:
return True

# Check Environment Variable
return bool(env_var_setting)
2 changes: 2 additions & 0 deletions packages/google-api-core/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -94,4 +94,6 @@ filterwarnings = [
"ignore:.*custom tp_new.*in Python 3.14:DeprecationWarning",
# Remove once https://github.com/grpc/grpc/issues/35086 is fixed (and version newer than 1.60.0 is published)
"ignore:There is no current event loop:DeprecationWarning",
# Ignore external OpenTelemetry/importlib.metadata SelectableGroups warning
"ignore:.*SelectableGroups dict interface is deprecated:DeprecationWarning",
Comment thread
chalmerlowe marked this conversation as resolved.
Outdated
]
1 change: 1 addition & 0 deletions packages/google-api-core/testing/constraints-3.10.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ requests==2.33.0
grpcio==1.41.0
grpcio-status==1.41.0
proto-plus==1.24.0
opentelemetry-api==1.27.0
Comment thread
chalmerlowe marked this conversation as resolved.
Outdated
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ grpcio==1.41.0
grpcio-status==1.41.0
proto-plus==1.24.0
aiohttp==3.13.4
opentelemetry-api==1.27.0
Comment thread
chalmerlowe marked this conversation as resolved.
Outdated
199 changes: 199 additions & 0 deletions packages/google-api-core/tests/unit/test_feature_gating_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
from google.api_core import feature_gating_helpers
from google.api_core.feature_gating_helpers import (
_get_env_bool,
_strtobool,
)


@pytest.mark.parametrize(
"value,expected",
[
# Truthy values
("y", True),
("yes", True),
("t", True),
("true", True),
("on", True),
("1", True),
(" True ", True),
# Falsy values
("n", False),
("no", False),
("f", False),
("false", False),
("off", False),
("0", False),
(" FALSE ", False),
# Empty string
("", None),
],
)
def test_strtobool(value, expected):
assert _strtobool(value) is expected


def test_strtobool_invalid():
with pytest.raises(ValueError):
_strtobool("invalid")


def test_get_env_bool(monkeypatch):
monkeypatch.setenv("TEST_VAR", "true")
assert _get_env_bool("TEST_VAR") is True

monkeypatch.setenv("TEST_VAR", "invalid")
import pytest

with pytest.warns(RuntimeWarning, match="Ignored invalid value"):
assert _get_env_bool("TEST_VAR") is None

monkeypatch.delenv("TEST_VAR", raising=False)
assert _get_env_bool("TEST_VAR") is None


def test_resolve_feature_flags_ga_enabled_via_env(monkeypatch):
"""Verify that a GA feature is enabled if its environment variable is True."""
# Setup: We pass a GA environment variable set to True
monkeypatch.setenv("GOOGLE_SDK_PYTHON_TRACING_ENABLED", "true")

# Action
result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=None,
)

# Assertion
assert result is True


@pytest.mark.parametrize("exp_env_state", [None, "false"], ids=["missing", "disabled"])
def test_resolve_feature_flags_exp_blocked_with_provider_fails_fast(
monkeypatch, exp_env_state
):
"""Verify that passing a provider to an experimental feature raises ValueError if the experimental environment variable is disabled or missing."""
# Setup: Experimental env var is set to exp_env_state (None means not set)
if exp_env_state is not None:
monkeypatch.setenv(
"GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED", exp_env_state
)
else:
monkeypatch.delenv(
"GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED", raising=False
)
client_options = {"tracer_provider": object()}

# Action & Assertion
with pytest.raises(ValueError, match="Experimental feature"):
feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=client_options,
)


def test_resolve_feature_flags_exp_enabled_with_provider(monkeypatch):
"""Verify that experimental feature is enabled if the experimental environment variable is enabled and a provider is provided."""
monkeypatch.setenv("GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED", "true")
client_options = {"tracer_provider": object()}

result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=client_options,
)
assert result is True


def test_resolve_feature_flags_exp_enabled_without_provider(monkeypatch):
"""Verify that experimental feature is enabled if the experimental environment variable is enabled and NO provider is provided."""
monkeypatch.setenv("GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED", "true")

result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=None,
)
assert result is True


def test_resolve_feature_flags_exp_disabled_without_provider(monkeypatch):
"""Verify that experimental feature is disabled if the experimental environment variable is disabled and NO provider is provided."""
monkeypatch.setenv("GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED", "false")

result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=None,
)
assert result is False


def test_resolve_feature_flags_ga_enabled_via_provider(monkeypatch):
"""Verify that a GA feature is enabled if a provider is provided, ignoring the environment variable."""
# Env var is False, but provider is present
monkeypatch.setenv("GOOGLE_SDK_PYTHON_TRACING_ENABLED", "false")
client_options = {"tracer_provider": object()}

result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=client_options,
)
assert result is True


@pytest.mark.parametrize(
"env_val", [None, "false"], ids=["env_not_set", "env_explicit_false"]
)
def test_resolve_feature_flags_ga_fallback_to_false(monkeypatch, env_val):
"""Verify that a GA feature is disabled if neither a provider is provided nor the environment variable is enabled."""
if env_val is not None:
monkeypatch.setenv("GOOGLE_SDK_PYTHON_TRACING_ENABLED", env_val)
else:
monkeypatch.delenv("GOOGLE_SDK_PYTHON_TRACING_ENABLED", raising=False)
result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=None,
)
assert result is False


class _MockOptions:
def __init__(self):
self.other_option = "value"


@pytest.mark.parametrize(
"client_options",
[
{"other_option": "value"},
_MockOptions(),
],
ids=["dict_without_key", "object_without_key"],
)
def test_resolve_feature_flags_options_without_key(client_options):
"""Verify behavior when client_options is present but missing the provider key."""
# GA Path: should fall through to env var / fallback
result = feature_gating_helpers.resolve_feature_flags(
env_var="GOOGLE_SDK_PYTHON_TRACING_ENABLED",
provider_key="tracer_provider",
client_options=client_options,
)
assert result is False
Loading