Skip to content

Commit 9c48260

Browse files
committed
feat: delegate boilerplate helpers to google-api-core
1 parent e0f3f85 commit 9c48260

96 files changed

Lines changed: 473 additions & 3598 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/gapic-generator-tests.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ jobs:
8585
curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
8686
cd /usr/src/protoc/ && unzip protoc.zip
8787
sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
88+
- name: Configure Git Redirect
89+
run: |
90+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
91+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
8892
- name: Run Nox
8993
env:
9094
GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
@@ -126,6 +130,10 @@ jobs:
126130
python-version: ${{ needs.python_config.outputs.latest_stable_python }}
127131
- name: Install System Deps
128132
run: sudo apt-get update && sudo apt-get install -y pandoc
133+
- name: Configure Git Redirect
134+
run: |
135+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
136+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
129137
- name: Run Goldens
130138
env:
131139
LATEST_STABLE_PYTHON: ${{ needs.python_config.outputs.latest_stable_python }}
@@ -201,6 +209,10 @@ jobs:
201209
curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
202210
cd /usr/src/protoc/ && unzip protoc.zip
203211
sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
212+
- name: Configure Git Redirect
213+
run: |
214+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
215+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
204216
- name: Run Tests
205217
run: |
206218
pip install nox
@@ -232,6 +244,10 @@ jobs:
232244
# Ensure CACHE_VERSION is defined in the mono-repo secrets!
233245
key: ${{ runner.os }}-bazel-20210105-${{ secrets.CACHE_VERSION }}
234246

247+
- name: Configure Git Redirect
248+
run: |
249+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
250+
git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
235251
- name: Run Bazel Integration Tests
236252
run: |
237253
# We need to move into the package directory if the

packages/gapic-generator/WORKSPACE

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ load("@rules_python//python:pip.bzl", "pip_parse")
4242

4343
pip_parse(
4444
name = "gapic_generator_python_pip_deps",
45-
requirements_lock = "//:requirements.txt",
45+
requirements_lock = "//:requirements.txt",
46+
extra_pip_args = ["--index-url=https://pypi.org/simple"],
4647
)
4748
load("@gapic_generator_python_pip_deps//:requirements.bzl", "install_deps")
4849

@@ -123,3 +124,13 @@ switched_rules_by_language(
123124
gapic = True,
124125
grpc = True,
125126
)
127+
128+
# BEGIN workaround
129+
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
130+
131+
python_register_toolchains(
132+
name = "python311",
133+
python_version = "3.11",
134+
)
135+
# END workaround
136+

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2

Lines changed: 36 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@
66
{% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %}
77
{% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
88

9+
{% set ns = namespace(has_auto_populated=false) %}
10+
{% for method in service.methods.values() %}
11+
{% set method_settings = api.all_method_settings.get(method.meta.address.proto) %}
12+
{% if method_settings is not none and method_settings.auto_populated_fields %}
13+
{% set ns.has_auto_populated = true %}
14+
{% endif %}
15+
{% endfor %}
16+
917
from collections import OrderedDict
1018
{% if service.any_extended_operations_methods %}
1119
import functools
@@ -16,8 +24,8 @@ import logging as std_logging
1624
import os
1725
import re
1826
from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast
19-
{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
20-
import uuid
27+
{% if ns.has_auto_populated %}
28+
from google.api_core.gapic_v1 import _method_helpers
2129
{% endif %}
2230
import warnings
2331

@@ -30,6 +38,9 @@ from google.api_core import exceptions as core_exceptions
3038
from google.api_core import extended_operation
3139
{% endif %}
3240
from google.api_core import gapic_v1
41+
from google.api_core.gapic_v1 import _client_cert
42+
from google.api_core.gapic_v1 import _config_helpers
43+
from google.api_core.gapic_v1 import _routing
3344
from google.api_core import retry as retries
3445
from google.auth import credentials as ga_credentials # type: ignore
3546
from google.auth.transport import mtls # type: ignore
@@ -154,28 +165,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
154165
Returns:
155166
Optional[str]: converted mTLS api endpoint.
156167
"""
157-
if not api_endpoint:
158-
return api_endpoint
159-
160-
mtls_endpoint_re = re.compile(
161-
r"(?P<name>[^.]+)(?P<mtls>\.mtls)?(?P<sandbox>\.sandbox)?(?P<googledomain>\.googleapis\.com)?"
162-
)
163-
164-
m = mtls_endpoint_re.match(api_endpoint)
165-
if m is None:
166-
# Could not parse api_endpoint; return as-is.
167-
return api_endpoint
168-
169-
name, mtls, sandbox, googledomain = m.groups()
170-
if mtls or not googledomain:
171-
return api_endpoint
172-
173-
if sandbox:
174-
return api_endpoint.replace(
175-
"sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
176-
)
177-
178-
return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
168+
return _routing.get_default_mtls_endpoint(api_endpoint)
179169

180170
# Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead.
181171
DEFAULT_ENDPOINT = {% if service.host %}"{{ service.host }}"{% else %}None{% endif %}
@@ -201,18 +191,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
201191
ValueError: (If using a version of google-auth without should_use_client_cert and
202192
GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.)
203193
"""
204-
# check if google-auth version supports should_use_client_cert for automatic mTLS enablement
205-
if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER
206-
return mtls.should_use_client_cert()
207-
else: # pragma: NO COVER
208-
# if unsupported, fallback to reading from env var
209-
use_client_cert_str = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower()
210-
if use_client_cert_str not in ("true", "false"):
211-
raise ValueError(
212-
"Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be"
213-
" either `true` or `false`"
214-
)
215-
return use_client_cert_str == "true"
194+
return _client_cert.use_client_cert_effective()
216195

217196
@classmethod
218197
def from_service_account_info(cls, info: dict, *args, **kwargs):
@@ -365,12 +344,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
365344
google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT
366345
is not any of ["auto", "never", "always"].
367346
"""
368-
use_client_cert = {{ service.client_name }}._use_client_cert_effective()
369-
use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower()
370-
universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN")
371-
if use_mtls_endpoint not in ("auto", "never", "always"):
372-
raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`")
373-
return use_client_cert, use_mtls_endpoint, universe_domain_env
347+
return _config_helpers.read_environment_variables()
374348

375349
@staticmethod
376350
def _get_client_cert_source(provided_cert_source, use_cert_flag):
@@ -383,13 +357,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
383357
Returns:
384358
bytes or None: The client cert source to be used by the client.
385359
"""
386-
client_cert_source = None
387-
if use_cert_flag:
388-
if provided_cert_source:
389-
client_cert_source = provided_cert_source
390-
elif mtls.has_default_client_cert_source():
391-
client_cert_source = mtls.default_client_cert_source()
392-
return client_cert_source
360+
return _client_cert.get_client_cert_source(provided_cert_source, use_cert_flag)
393361

394362
@staticmethod
395363
def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint) -> str:
@@ -406,16 +374,18 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
406374
Returns:
407375
str: The API endpoint to be used by the client.
408376
"""
409-
if api_override is not None:
410-
api_endpoint = api_override
411-
elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source):
412-
_default_universe = {{ service.client_name }}._DEFAULT_UNIVERSE
413-
if universe_domain != _default_universe:
414-
raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.")
415-
api_endpoint = {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT
416-
else:
417-
api_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain)
418-
return api_endpoint
377+
return cast(
378+
str,
379+
_routing.get_api_endpoint(
380+
api_override,
381+
client_cert_source,
382+
universe_domain,
383+
use_mtls_endpoint,
384+
{{ service.client_name }}._DEFAULT_UNIVERSE,
385+
{{ service.client_name }}.DEFAULT_MTLS_ENDPOINT,
386+
{{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE,
387+
),
388+
)
419389

420390
@staticmethod
421391
def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str:
@@ -431,14 +401,11 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
431401
Raises:
432402
ValueError: If the universe domain is an empty string.
433403
"""
434-
universe_domain = {{ service.client_name }}._DEFAULT_UNIVERSE
435-
if client_universe_domain is not None:
436-
universe_domain = client_universe_domain
437-
elif universe_domain_env is not None:
438-
universe_domain = universe_domain_env
439-
if len(universe_domain.strip()) == 0:
440-
raise ValueError("Universe Domain cannot be an empty string.")
441-
return universe_domain
404+
return _routing.get_universe_domain(
405+
client_universe_domain,
406+
universe_domain_env,
407+
{{ service.client_name }}._DEFAULT_UNIVERSE,
408+
)
442409

443410
def _validate_universe_domain(self):
444411
"""Validates client's and credentials' universe domains are consistent.
@@ -453,7 +420,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
453420
# NOTE (b/349488459): universe validation is disabled until further notice.
454421
return True
455422

456-
{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
423+
{% if ns.has_auto_populated %}
457424
@staticmethod
458425
def _setup_request_id(request, field_name: str, is_proto3_optional: bool):
459426
"""Populate a UUID4 field in the request if it is not already set.
@@ -463,26 +430,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
463430
field_name (str): The name of the field to populate.
464431
is_proto3_optional (bool): Whether the field is proto3 optional.
465432
"""
466-
if isinstance(request, dict):
467-
if is_proto3_optional:
468-
if field_name not in request:
469-
request[field_name] = str(uuid.uuid4())
470-
elif not request.get(field_name):
471-
request[field_name] = str(uuid.uuid4())
472-
return
473-
474-
if is_proto3_optional:
475-
try:
476-
# Pure protobuf messages
477-
if not request.HasField(field_name):
478-
setattr(request, field_name, str(uuid.uuid4()))
479-
except (AttributeError, ValueError):
480-
# Proto-plus messages or other objects
481-
if field_name not in request:
482-
setattr(request, field_name, str(uuid.uuid4()))
483-
else:
484-
if not getattr(request, field_name):
485-
setattr(request, field_name, str(uuid.uuid4()))
433+
_method_helpers.setup_request_id(request, field_name, is_proto3_optional)
486434
{% endif %}
487435

488436
def _add_cred_info_for_auth_errors(

packages/gapic-generator/gapic/templates/setup.py.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ else:
3636
release_status = "Development Status :: 5 - Production/Stable"
3737

3838
dependencies = [
39-
"google-api-core[grpc] >= 2.25.0, <3.0.0",
39+
"google-api-core[grpc] @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core",
4040
# Exclude incompatible versions of `google-auth`
4141
# See https://github.com/googleapis/google-cloud-python/issues/12364
4242
"google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",

packages/gapic-generator/gapic/templates/testing/_default_constraints.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{% from '_pypi_packages.j2' import pypi_packages %}
22
# This constraints file is required for unit tests.
33
# List all library dependencies and extras in this file.
4-
google-api-core
4+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
55
google-auth
66
grpcio
77
proto-plus

packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
# pinning their versions to their lower bounds.
99
# For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
1010
# then this file should have google-cloud-foo==1.14.0
11-
google-api-core==2.25.0
11+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
1212
google-auth==2.35.0
1313
grpcio==1.59.0
1414
proto-plus==1.26.1

packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# pinning their versions to their lower bounds.
66
# For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
77
# then this file should have google-cloud-foo==1.14.0
8-
google-api-core==2.25.0
8+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
99
google-auth==2.14.1
1010
grpcio==1.59.0
1111
proto-plus==1.26.1

packages/gapic-generator/gapic/templates/testing/constraints-3.13.txt.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# Require the latest major version be installed for each dependency.
77
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
88
# Then this file should have google-cloud-foo>=1
9-
google-api-core>=2
9+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
1010
google-auth>=2
1111
grpcio>=1
1212
proto-plus>=1

packages/gapic-generator/gapic/templates/testing/constraints-3.14.txt.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# Require the latest major version be installed for each dependency.
77
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
88
# Then this file should have google-cloud-foo>=1
9-
google-api-core>=2
9+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
1010
google-auth>=2
1111
grpcio>=1
1212
proto-plus>=1

packages/gapic-generator/gapic/templates/testing/constraints-3.15.txt.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# Require the latest major version be installed for each dependency.
77
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
88
# Then this file should have google-cloud-foo>=1
9-
google-api-core>=2
9+
google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
1010
google-auth>=2
1111
grpcio>=1
1212
proto-plus>=1

0 commit comments

Comments
 (0)