Skip to content
This repository was archived by the owner on Mar 9, 2026. It is now read-only.

Commit 2039ef3

Browse files
partheaparthea
committed
wip librarian generation
1 parent 625c261 commit 2039ef3

3 files changed

Lines changed: 469 additions & 3 deletions

File tree

‎.librarian/generator-input/owlbot.py‎

Lines changed: 352 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,352 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import json
16+
from pathlib import Path
17+
import re
18+
import shutil
19+
import textwrap
20+
21+
import synthtool as s
22+
import synthtool.gcp as gcp
23+
from synthtool.languages import python
24+
25+
# ----------------------------------------------------------------------------
26+
# Copy the generated client from the owl-bot staging directory
27+
# ----------------------------------------------------------------------------
28+
29+
clean_up_generated_samples = True
30+
31+
# Load the default version defined in .repo-metadata.json.
32+
default_version = json.load(open(".repo-metadata.json", "rt")).get(
33+
"default_version"
34+
)
35+
36+
for library in s.get_staging_dirs(default_version):
37+
if clean_up_generated_samples:
38+
shutil.rmtree("samples/generated_samples", ignore_errors=True)
39+
clean_up_generated_samples = False
40+
41+
# DEFAULT SCOPES and SERVICE_ADDRESS are being used. so let's force them in.
42+
s.replace(
43+
library / f"google/pubsub_{library.name}/services/*er/*client.py",
44+
r"""DEFAULT_ENDPOINT = \"pubsub\.googleapis\.com\"""",
45+
"""
46+
# The scopes needed to make gRPC calls to all of the methods defined in
47+
# this service
48+
_DEFAULT_SCOPES = (
49+
'https://www.googleapis.com/auth/cloud-platform',
50+
'https://www.googleapis.com/auth/pubsub',
51+
)
52+
53+
SERVICE_ADDRESS = "pubsub.googleapis.com:443"
54+
\"""The default address of the service.\"""
55+
56+
\g<0>""",
57+
)
58+
59+
# Modify GRPC options in transports.
60+
count = s.replace(
61+
[
62+
library / f"google/pubsub_{library.name}/services/*/transports/grpc*",
63+
library / f"tests/unit/gapic/pubsub_{library.name}/*",
64+
],
65+
"options=\[.*?\]",
66+
"""options=[
67+
("grpc.max_send_message_length", -1),
68+
("grpc.max_receive_message_length", -1),
69+
("grpc.max_metadata_size", 4 * 1024 * 1024),
70+
("grpc.keepalive_time_ms", 30000),
71+
]""",
72+
flags=re.MULTILINE | re.DOTALL,
73+
)
74+
75+
if count < 15:
76+
raise Exception("Expected replacements for gRPC channel options not made.")
77+
78+
# If the emulator is used, force an insecure gRPC channel to avoid SSL errors.
79+
clients_to_patch = [
80+
library / f"google/pubsub_{library.name}/services/publisher/client.py",
81+
library / f"google/pubsub_{library.name}/services/subscriber/client.py",
82+
library / f"google/pubsub_{library.name}/services/schema_service/client.py",
83+
]
84+
err_msg = (
85+
"Expected replacements for gRPC channel to use with the emulator not made."
86+
)
87+
88+
count = s.replace(clients_to_patch, r"import os", "import functools\n\g<0>")
89+
90+
if count < len(clients_to_patch):
91+
raise Exception(err_msg)
92+
93+
count = s.replace(
94+
clients_to_patch,
95+
f"from \.transports\.base",
96+
"\nimport grpc\n\g<0>",
97+
)
98+
99+
if count < len(clients_to_patch):
100+
raise Exception(err_msg)
101+
102+
# TODO(https://github.com/googleapis/python-pubsub/issues/1349): Move the emulator
103+
# code below to test files.
104+
count = s.replace(
105+
clients_to_patch,
106+
r"# initialize with the provided callable or the passed in class",
107+
"""\g<0>
108+
109+
emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST")
110+
if emulator_host:
111+
if issubclass(transport_init, type(self)._transport_registry["grpc"]): # type: ignore
112+
channel = grpc.insecure_channel(target=emulator_host)
113+
else:
114+
channel = grpc.aio.insecure_channel(target=emulator_host)
115+
transport_init = functools.partial(transport_init, channel=channel)
116+
117+
""",
118+
)
119+
120+
if count < len(clients_to_patch):
121+
raise Exception(err_msg)
122+
123+
# Monkey patch the streaming_pull() GAPIC method to disable pre-fetching stream
124+
# results.
125+
s.replace(
126+
library / f"google/pubsub_{library.name}/services/subscriber/client.py",
127+
(
128+
r"# Wrap the RPC method.*\n"
129+
r"\s+# and friendly error.*\n"
130+
r"\s+rpc = self\._transport\._wrapped_methods\[self\._transport\.streaming_pull\]"
131+
),
132+
"""
133+
# Wrappers in api-core should not automatically pre-fetch the first
134+
# stream result, as this breaks the stream when re-opening it.
135+
# https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
136+
self._transport.streaming_pull._prefetch_first_result_ = False
137+
138+
\g<0>""",
139+
)
140+
141+
# Emit deprecation warning if return_immediately flag is set with synchronous pull.
142+
s.replace(
143+
library / f"google/pubsub_{library.name}/services/subscriber/*client.py",
144+
r"from google.pubsub_v1 import gapic_version as package_version",
145+
"import warnings\n\g<0>",
146+
)
147+
148+
count = s.replace(
149+
library / f"google/pubsub_{library.name}/services/subscriber/*client.py",
150+
r"""
151+
([^\n\S]+(?:async\ )?def\ pull\(.*?->\ pubsub\.PullResponse:.*?)
152+
((?P<indent>[^\n\S]+)\#\ Wrap\ the\ RPC\ method)
153+
""",
154+
textwrap.dedent(
155+
"""
156+
\g<1>
157+
\g<indent>if request.return_immediately:
158+
\g<indent> warnings.warn(
159+
\g<indent> "The return_immediately flag is deprecated and should be set to False.",
160+
\g<indent> category=DeprecationWarning,
161+
\g<indent> )
162+
163+
\g<2>"""
164+
),
165+
flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
166+
)
167+
168+
if count != 2:
169+
raise Exception("Too many or too few replacements in pull() methods.")
170+
171+
# Silence deprecation warnings in pull() method flattened parameter tests.
172+
s.replace(
173+
library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
174+
"import os",
175+
"\g<0>\nimport warnings",
176+
)
177+
178+
count = s.replace(
179+
library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
180+
textwrap.dedent(
181+
r"""
182+
([^\n\S]+# Call the method with a truthy value for each flattened field,
183+
[^\n\S]+# using the keyword arguments to the method\.)
184+
\s+(client\.pull\(.*?\))"""
185+
),
186+
"""\n\g<1>
187+
with warnings.catch_warnings():
188+
warnings.simplefilter("ignore", category=DeprecationWarning)
189+
\g<2>""",
190+
flags=re.MULTILINE | re.DOTALL,
191+
)
192+
193+
if count < 1:
194+
raise Exception("Catch warnings replacement failed.")
195+
196+
count = s.replace(
197+
library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
198+
textwrap.dedent(
199+
r"""
200+
([^\n\S]+# Call the method with a truthy value for each flattened field,
201+
[^\n\S]+# using the keyword arguments to the method\.)
202+
\s+response = (await client\.pull\(.*?\))"""
203+
),
204+
"""\n\g<1>
205+
with warnings.catch_warnings():
206+
warnings.simplefilter("ignore", category=DeprecationWarning)
207+
\g<2>""",
208+
flags=re.MULTILINE | re.DOTALL,
209+
)
210+
211+
if count < 1:
212+
raise Exception("Catch warnings replacement failed.")
213+
214+
# Make sure that client library version is present in user agent header.
215+
count = s.replace(
216+
[
217+
library
218+
/ f"google/pubsub_{library.name}/services/publisher/async_client.py",
219+
library / f"google/pubsub_{library.name}/services/publisher/client.py",
220+
library
221+
/ f"google/pubsub_{library.name}/services/publisher/transports/base.py",
222+
library
223+
/ f"google/pubsub_{library.name}/services/schema_service/async_client.py",
224+
library / f"google/pubsub_{library.name}/services/schema_service/client.py",
225+
library
226+
/ f"google/pubsub_{library.name}/services/schema_service/transports/base.py",
227+
library
228+
/ f"google/pubsub_{library.name}/services/subscriber/async_client.py",
229+
library / f"google/pubsub_{library.name}/services/subscriber/client.py",
230+
library
231+
/ f"google/pubsub_{library.name}/services/subscriber/transports/base.py",
232+
],
233+
r"""gapic_version=package_version.__version__""",
234+
"client_library_version=package_version.__version__",
235+
)
236+
237+
if count < 1:
238+
raise Exception("client_library_version replacement failed.")
239+
240+
# Allow timeout to be an instance of google.api_core.timeout.*
241+
count = s.replace(
242+
library / f"google/pubsub_{library.name}/types/__init__.py",
243+
r"from \.pubsub import \(",
244+
"from typing import Union\n\n\g<0>",
245+
)
246+
247+
if count < 1:
248+
raise Exception("Catch timeout replacement 1 failed.")
249+
250+
count = s.replace(
251+
library / f"google/pubsub_{library.name}/types/__init__.py",
252+
r"__all__ = \(\n",
253+
textwrap.dedent(
254+
'''\
255+
TimeoutType = Union[
256+
int,
257+
float,
258+
"google.api_core.timeout.ConstantTimeout",
259+
"google.api_core.timeout.ExponentialTimeout",
260+
]
261+
"""The type of the timeout parameter of publisher client methods."""
262+
263+
\g<0> "TimeoutType",'''
264+
),
265+
)
266+
267+
if count < 1:
268+
raise Exception("Catch timeout replacement 2 failed.")
269+
270+
count = s.replace(
271+
library / f"google/pubsub_{library.name}/services/publisher/*client.py",
272+
r"from google.api_core import retry as retries.*\n",
273+
"\g<0>from google.api_core import timeout as timeouts # type: ignore\n",
274+
)
275+
276+
if count < 1:
277+
raise Exception("Catch timeout replacement 3 failed.")
278+
279+
count = s.replace(
280+
library / f"google/pubsub_{library.name}/services/publisher/*client.py",
281+
f"from google\.pubsub_{library.name}\.types import pubsub",
282+
f"\g<0>\nfrom google.pubsub_{library.name}.types import TimeoutType",
283+
)
284+
285+
if count < 1:
286+
raise Exception("Catch timeout replacement 4 failed.")
287+
288+
count = s.replace(
289+
library / f"google/pubsub_{library.name}/services/publisher/*client.py",
290+
r"(\s+)timeout: Union\[float, object\] = gapic_v1.method.DEFAULT.*\n",
291+
f"\g<1>timeout: TimeoutType = gapic_{library.name}.method.DEFAULT,",
292+
)
293+
294+
if count < 1:
295+
raise Exception("Catch timeout replacement 5 failed.")
296+
297+
count = s.replace(
298+
library / f"google/pubsub_{library.name}/services/publisher/*client.py",
299+
r"([^\S\r\n]+)timeout \(float\): (.*)\n",
300+
("\g<1>timeout (TimeoutType):\n" "\g<1> \g<2>\n"),
301+
)
302+
303+
if count < 1:
304+
raise Exception("Catch timeout replacement 6 failed.")
305+
306+
# Override the default max retry deadline for publisher methods.
307+
count = s.replace(
308+
library / f"google/pubsub_{library.name}/services/publisher/transports/base.py",
309+
r"deadline=60\.0",
310+
"deadline=600.0",
311+
)
312+
if count < 9:
313+
raise Exception(
314+
"Default retry deadline not overriden for all publisher methods."
315+
)
316+
317+
# The namespace package declaration in google/cloud/__init__.py should be excluded
318+
# from coverage.
319+
count = s.replace(
320+
library / ".coveragerc",
321+
"google/pubsub/__init__.py",
322+
"""google/cloud/__init__.py
323+
google/pubsub/__init__.py""",
324+
)
325+
326+
if count < 1:
327+
raise Exception(".coveragerc replacement failed.")
328+
329+
s.move([library], excludes=["**/gapic_version.py", "noxfile.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt", "testing/constraints-3.8.txt"])
330+
s.remove_staging_dirs()
331+
332+
# ----------------------------------------------------------------------------
333+
# Add templated files
334+
# ----------------------------------------------------------------------------
335+
336+
templated_files = gcp.CommonTemplates().py_library(
337+
microgenerator=True,
338+
samples=True,
339+
cov_level=99,
340+
versions=gcp.common.detect_versions(path="./google", default_first=True),
341+
unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"],
342+
unit_test_dependencies=["flaky"],
343+
system_test_python_versions=["3.12"],
344+
system_test_external_dependencies=["psutil","flaky"],
345+
)
346+
s.move(templated_files, excludes=[".coveragerc", ".github/**", "README.rst", "docs/**", ".kokoro/**"])
347+
348+
python.py_samples(skip_readmes=True)
349+
350+
# run format session for all directories which have a noxfile
351+
for noxfile in Path(".").glob("**/noxfile.py"):
352+
s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False)

0 commit comments

Comments
 (0)