Skip to content

Commit f4246dc

Browse files
chalmerlowechalmerlowe
committed
feat: prepare handwritten libraries for easy mypy activation
1 parent 97a3be9 commit f4246dc

9 files changed

Lines changed: 236 additions & 22 deletions

File tree

‎.librarian/generator-input/client-post-processing/bigtable-integration.yaml‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ replacements:
403403
[mypy-google.cloud.bigtable.data.*]
404404
ignore_errors = False
405405
count: 1
406-
# Note: noxfile.py is heavily customized so we clobber the whole file.
406+
# Note: noxfile.py is heavily customized so we clobber the whole file.
407407
- paths: [
408408
packages/google-cloud-bigtable/noxfile.py
409409
]
@@ -464,6 +464,16 @@ replacements:
464464
SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}
465465
466466
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
467+
# Path to the centralized mypy configuration file at the repository root.
468+
# Search upwards to support running nox from both monorepo packages and integration test goldens.
469+
MYPY_CONFIG_FILE = next(
470+
(
471+
str(p / "mypy.ini")
472+
for p in CURRENT_DIRECTORY.parents
473+
if (p / "mypy.ini").exists()
474+
),
475+
str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
476+
)
467477
468478
# 'docfx' is excluded since it only needs to run in 'docs-presubmit'
469479
nox.options.sessions = [
@@ -573,7 +583,12 @@ replacements:
573583
"types-requests",
574584
)
575585
session.install("google-cloud-testutils")
576-
session.run("mypy", "-p", "google.cloud.bigtable.data")
586+
session.run(
587+
"mypy",
588+
f"--config-file={MYPY_CONFIG_FILE}",
589+
"-p",
590+
"google.cloud.bigtable.data",
591+
)
577592
578593
579594
@nox.session(python=DEFAULT_PYTHON_VERSION)

‎.librarian/generator-input/client-post-processing/logging-integration.yaml‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -328,14 +328,14 @@ replacements:
328328
-------------------
329329
.. toctree::
330330
:maxdepth: 2
331-
331+
332332
usage
333-
333+
334334
Documentation
335335
-------------------
336336
.. toctree::
337337
:maxdepth: 3
338-
338+
339339
client
340340
logger
341341
entries
@@ -344,20 +344,20 @@ replacements:
344344
sink
345345
handlers
346346
transport
347-
348-
347+
348+
349349
Migration Guides
350350
----------------
351-
351+
352352
See the guide below for instructions on migrating between major releases of this library.
353-
353+
354354
.. toctree::
355355
:maxdepth: 2
356-
356+
357357
UPGRADING
358358
count: 1
359359
- paths: [
360-
"packages/google-cloud-logging/google/cloud/logging/__init__.py",
360+
"packages/google-cloud-logging/google/cloud/logging/__init__.py",
361361
]
362362
before: |
363363
from google.cloud.logging import gapic_version as package_version
@@ -463,6 +463,24 @@ replacements:
463463
"django",
464464
]
465465
count: 1
466+
- paths: [
467+
packages/google-cloud-logging/noxfile.py
468+
]
469+
before: |
470+
CURRENT_DIRECTORY = pathlib.Path\(__file__\).parent.absolute\(\)
471+
after: |
472+
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
473+
# Path to the centralized mypy configuration file at the repository root.
474+
# Search upwards to support running nox from both monorepo packages and integration test goldens.
475+
MYPY_CONFIG_FILE = next(
476+
(
477+
str(p / "mypy.ini")
478+
for p in CURRENT_DIRECTORY.parents
479+
if (p / "mypy.ini").exists()
480+
),
481+
str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
482+
)
483+
count: 1
466484
- paths: [
467485
packages/google-cloud-logging/noxfile.py
468486
]
@@ -478,4 +496,19 @@ replacements:
478496
# TODO(https://github.com/googleapis/google-cloud-python/issues/13362):
479497
# Enable mypy once this repo has been updated for mypy evaluation.
480498
session.skip("Skip mypy since this library is not yet updated for mypy evaluation")
499+
500+
session.install("-e", ".")
501+
session.install(
502+
"mypy",
503+
"types-setuptools",
504+
"types-protobuf",
505+
"types-requests",
506+
)
507+
session.run(
508+
"mypy",
509+
f"--config-file={MYPY_CONFIG_FILE}",
510+
"-p",
511+
"google",
512+
*session.posargs,
513+
)
481514
count: 1

‎.librarian/generator-input/client-post-processing/pubsub-integration.yaml‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
replacements:
2-
- paths:
2+
- paths:
33
- "packages/google-cloud-pubsub/google/pubsub_v1/services/**/*client.py"
44
before: 'instead\.\n(?:[ \t]*\n)*([ \t]+)DEFAULT_ENDPOINT = "pubsub\.googleapis\.com"'
55
after: |-
@@ -13,7 +13,7 @@ replacements:
1313
\g<1>DEFAULT_ENDPOINT = "pubsub.googleapis.com"
1414
count: 3
1515

16-
- paths:
16+
- paths:
1717
- "packages/google-cloud-pubsub/google/pubsub_v1/services/*/client.py"
1818
before: 'import json\n(?:[ \t]*\n)*import logging as std_logging'
1919
after: |-
@@ -23,7 +23,7 @@ replacements:
2323
import logging as std_logging
2424
count: 3
2525

26-
- paths:
26+
- paths:
2727
- "packages/google-cloud-pubsub/google/pubsub_v1/services/*/client.py"
2828
before: 'class\n(?:[ \t]*\n)*([ \t]+)self\._transport = transport_init\('
2929
after: |-
@@ -38,7 +38,7 @@ replacements:
3838
\g<1>self._transport = transport_init(
3939
count: 3
4040

41-
- paths:
41+
- paths:
4242
- "packages/google-cloud-pubsub/google/pubsub_v1/services/*/transports/grpc*.py"
4343
- "packages/google-cloud-pubsub/tests/unit/gapic/pubsub_v1/*.py"
4444
before: '\("grpc\.max_receive_message_length", -1\),\n(?:[ \t]*\n)*([ \t]+)\]'
@@ -49,19 +49,19 @@ replacements:
4949
\g<1>]
5050
count: 21
5151

52-
- paths:
52+
- paths:
5353
- "packages/google-cloud-pubsub/google/pubsub_v1/services/publisher/transports/base.py"
5454
before: 'deadline=60\.0,'
5555
after: 'deadline=600.0,'
5656
count: 9
5757

58-
- paths:
58+
- paths:
5959
- "packages/google-cloud-pubsub/google/pubsub_v1/services/**/*.py"
6060
before: 'gapic_version=package_version\.__version__'
6161
after: 'client_library_version=package_version.__version__'
6262
count: 9
6363

64-
- paths:
64+
- paths:
6565
- "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
6666
before: 'import logging as std_logging\n(?:[ \t]*\n)*import os'
6767
after: |-
@@ -71,7 +71,7 @@ replacements:
7171
import os
7272
count: 1
7373

74-
- paths:
74+
- paths:
7575
- "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/async_client.py"
7676
before: 'import logging as std_logging\n(?:[ \t]*\n)*import re'
7777
after: |-
@@ -80,7 +80,7 @@ replacements:
8080
import re
8181
count: 1
8282

83-
- paths:
83+
- paths:
8484
- "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
8585
- "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/async_client.py"
8686
before: 'request\.max_messages = max_messages\n(?:[ \t]*\n)*([ \t]+)# Wrap the RPC method'
@@ -96,7 +96,7 @@ replacements:
9696
\g<1># Wrap the RPC method
9797
count: 2
9898

99-
- paths:
99+
- paths:
100100
- "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
101101
before: '"""\n(?:[ \t]*\n)*([ \t]+)# Wrap the RPC method; this adds retry and timeout information,\n(?:[ \t]*\n)*[ \t]+# and friendly error handling\.\n(?:[ \t]*\n)*[ \t]+rpc = self\._transport\._wrapped_methods\[self\._transport\.streaming_pull\]'
102102
after: |-
@@ -403,6 +403,16 @@ replacements:
403403
before: 'nox\.options\.sessions = \[\n(?:[ \t]+)"unit",\n(?:[ \t]+)"system",\n(?:[ \t]+)"cover",\n(?:[ \t]+)"lint",\n(?:[ \t]+)"lint_setup_py",\n(?:[ \t]+)"blacken",\n(?:[ \t]+)"docs",\n\]'
404404
after: |-
405405
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
406+
# Path to the centralized mypy configuration file at the repository root.
407+
# Search upwards to support running nox from both monorepo packages and integration test goldens.
408+
MYPY_CONFIG_FILE = next(
409+
(
410+
str(p / "mypy.ini")
411+
for p in CURRENT_DIRECTORY.parents
412+
if (p / "mypy.ini").exists()
413+
),
414+
str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
415+
)
406416
407417
nox.options.sessions = [
408418
"unit",
@@ -450,7 +460,7 @@ replacements:
450460
# mypy checks yet.
451461
# https://github.com/googleapis/gapic-generator-python/issues/1092
452462
# TODO: Re-enable mypy checks once we merge, since incremental checks are failing due to protobuf upgrade
453-
# session.run("mypy", "-p", "google.cloud", "--exclude", "google/pubsub_v1/")
463+
# session.run("mypy", f"--config-file={MYPY_CONFIG_FILE}", "-p", "google.cloud", "--exclude", "google/pubsub_v1/")
454464
455465
456466
@nox.session(python=DEFAULT_PYTHON_VERSION)

‎.librarian/generator-input/client-post-processing/storage-integration.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,23 @@
11
description: Restore handwritten __init__.py for google-cloud-storage
22
replacements:
3+
- paths: [
4+
packages/google-cloud-storage/noxfile.py,
5+
]
6+
before: |
7+
CURRENT_DIRECTORY = pathlib.Path\(__file__\).parent.absolute\(\)
8+
after: |
9+
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
10+
# Path to the centralized mypy configuration file at the repository root.
11+
# Search upwards to support running nox from both monorepo packages and integration test goldens.
12+
MYPY_CONFIG_FILE = next(
13+
(
14+
str(p / "mypy.ini")
15+
for p in CURRENT_DIRECTORY.parents
16+
if (p / "mypy.ini").exists()
17+
),
18+
str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
19+
)
20+
count: 1
321
- paths: [
422
packages/google-cloud-storage/noxfile.py,
523
]
@@ -804,4 +822,19 @@ replacements:
804822
# TODO(https://github.com/googleapis/google-cloud-python/issues/13362):
805823
# Enable mypy once this repo has been updated for mypy evaluation.
806824
session.skip("Skip mypy since this library is not yet updated for mypy evaluation")
825+
826+
session.install("-e", ".")
827+
session.install(
828+
"mypy",
829+
"types-setuptools",
830+
"types-protobuf",
831+
"types-requests",
832+
)
833+
session.run(
834+
"mypy",
835+
f"--config-file={MYPY_CONFIG_FILE}",
836+
"-p",
837+
"google",
838+
*session.posargs,
839+
)
807840
count: 1

‎mypy_activation_readiness_plan.md‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Mypy Activation Readiness Plan 🏴󠁧󠁢󠁳󠁣󠁴󠁿
2+
3+
This plan outlines the steps to ensure all libraries in PR #17409 are prepared for easy Mypy activation. The goal is to have the `def mypy(session):` block fully populated with installation logic and the centralized `session.run` command, but kept inactive via `session.skip()` or commented-out code.
4+
5+
## 📋 Current State Summary
6+
- All PR files have `MYPY_CONFIG_FILE` defined.
7+
- Several packages (`logging`, `storage`, `proto-plus`, etc.) are skipped but **lack the session boilerplate** (installations + `session.run`).
8+
- Some packages are heavily modified by **Client Post-Processing** (YAMLs), meaning direct edits to `noxfile.py` will be clobbered.
9+
10+
---
11+
12+
## 🛠️ Action Plan
13+
14+
### Phase 1: Update Post-Processing YAMLs (Managed Handwritten Libraries)
15+
For libraries whose `noxfile.py` is generated or heavily modified by pipeline scripts, we must update the YAML files in `.librarian/generator-input/client-post-processing/`.
16+
17+
#### 1. `google-cloud-logging`
18+
- **File**: `logging-integration.yaml`
19+
- **Current**: Replaces the session body with `session.skip(...)`.
20+
- **Target**: Update the `after:` block to include standard Mypy installations and the `session.run` command (pointing to `MYPY_CONFIG_FILE`), with `session.skip` at the top of the body.
21+
22+
#### 2. `google-cloud-storage`
23+
- **File**: `storage-integration.yaml`
24+
- **Current**: Replaces the session body with `session.skip(...)`.
25+
- **Target**: Same as logging. Inject full boilerplate below the skip.
26+
27+
#### 3. `google-cloud-pubsub`
28+
- **File**: `pubsub-integration.yaml`
29+
- **Current**: Has boilerplate, but `session.run` is commented out.
30+
- **Target**: Keep commented out as requested (to respect convention), but ensure it uses `{MYPY_CONFIG_FILE}`. (✅ *Already updated in previous step*).
31+
32+
#### 4. `google-cloud-bigtable`
33+
- **File**: `bigtable-integration.yaml`
34+
- **Current**: Heavily customized, active session.
35+
- **Target**: Ensure it uses `{MYPY_CONFIG_FILE}`. (✅ *Already updated in previous step*).
36+
37+
---
38+
39+
### Phase 2: Update Static Handwritten Libraries (No Post-Processing)
40+
For libraries that are purely handwritten and not touched by the Librarian pipeline, we can edit `noxfile.py` directly.
41+
42+
#### 1. `proto-plus`
43+
- Populate `def mypy(session):` with standard installations and `session.run("mypy", f"--config-file={MYPY_CONFIG_FILE}", ...)` below the `session.skip` call.
44+
45+
#### 2. `bigquery-magics`
46+
- Populate session with boilerplate below `session.skip`.
47+
48+
#### 3. `django-google-spanner`
49+
- Populate session with boilerplate below `session.skip`.
50+
51+
#### 4. `gcp-sphinx-docfx-yaml`
52+
- Populate session with boilerplate below `session.skip`.
53+
54+
---
55+
56+
### Phase 3: Verification
57+
1. Run `nox -s mypy` (or equivalent) in a couple of updated packages to verify they skip correctly.
58+
2. Verify the `MYPY_CONFIG_FILE` path is correctly derived in various execution contexts (monorepo root vs package dir).
59+
60+
---
61+
62+
> [!TIP]
63+
> **Enabling Mypy Later**: Once this plan is executed, turning on Mypy for any of these packages is as simple as removing the `session.skip()` line or uncommenting the `session.run` command in the respective YAML or `noxfile.py`.

‎packages/bigquery-magics/noxfile.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -557,3 +557,18 @@ def mypy(session):
557557
# TODO(https://github.com/googleapis/google-cloud-python/issues/16014):
558558
# Add mypy tests
559559
session.skip("mypy tests are not yet supported")
560+
561+
session.install("-e", ".")
562+
session.install(
563+
"mypy",
564+
"types-setuptools",
565+
"types-protobuf",
566+
"types-requests",
567+
)
568+
session.run(
569+
"mypy",
570+
f"--config-file={MYPY_CONFIG_FILE}",
571+
"-p",
572+
"bigquery_magics",
573+
*session.posargs,
574+
)

‎packages/django-google-spanner/noxfile.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,21 @@ def mypy(session):
331331
# Add typehints to this package.
332332
session.skip("Typehints and thus mypy are not yet supported.")
333333

334+
session.install("-e", ".")
335+
session.install(
336+
"mypy",
337+
"types-setuptools",
338+
"types-protobuf",
339+
"types-requests",
340+
)
341+
session.run(
342+
"mypy",
343+
f"--config-file={MYPY_CONFIG_FILE}",
344+
"-p",
345+
"django_spanner",
346+
*session.posargs,
347+
)
348+
334349

335350
@nox.session(python=DEFAULT_PYTHON_VERSION)
336351
@nox.parametrize(

0 commit comments

Comments
 (0)