chore(handwritten): centralize mypy configuration and update handwritten package noxfiles (#17409)

* Relies on a centralized `mypy.ini` file at the root of the repository
to consolidate all shared and package-specific mypy exceptions.
* Deletes the individual `mypy.ini` files from handwritten libraries,
establishing the root configuration as the source of truth.
* Refactored their `noxfile.py` files to define `MYPY_CONFIG_FILE`
dynamically and standardized their `session.run("mypy", ...)`
invocations.

> [!note]
> Due to test failures outside the scope of this PR, neither
google-cloud-crc32c nor bigframes are included in this PR. This [issue
tracks those
packages](#17606)
to ensure their eventual completion

> [!note]
> bigquery and spanner were pulled out and placed in separate PRs due to
excessive duration of their tests. PR #17640 and PR #17641

> [!note]
> Work on generated libraries is outside the scope of this PR and can be
found here: #17408

> [!note]
> UPDATE (20260706): This PR is no longer blocked by
#17607

Author: chalmerlowe

.librarian/generator-input/client-post-processing/bigtable-integration.yaml

@@ -419,6 +419,16 @@ replacements:
       SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}
 
       CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+      # Path to the centralized mypy configuration file at the repository root.
+      # Search upwards to support running nox from both monorepo packages and integration test goldens.
+      MYPY_CONFIG_FILE = next(
+          (
+              str(p / "mypy.ini")
+              for p in CURRENT_DIRECTORY.parents
+              if (p / "mypy.ini").exists()
+          ),
+          str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
+      )
 
       # 'docfx' is excluded since it only needs to run in 'docs-presubmit'
       nox.options.sessions = [
@@ -528,7 +538,12 @@ replacements:
               "types-requests",
           )
           session.install("google-cloud-testutils")
-          session.run("mypy", "-p", "google.cloud.bigtable.data")
+          session.run(
+              "mypy",
+              f"--config-file={MYPY_CONFIG_FILE}",
+              "-p",
+              "google.cloud.bigtable.data",
+          )
 
 
       @nox.session(python=DEFAULT_PYTHON_VERSION)

.librarian/generator-input/client-post-processing/logging-integration.yaml

@@ -328,14 +328,14 @@ replacements:
       -------------------
       .. toctree::
          :maxdepth: 2
-      
+
          usage
-      
+
       Documentation
       -------------------
       .. toctree::
          :maxdepth: 3
-      
+
          client
          logger
          entries
@@ -344,20 +344,20 @@ replacements:
          sink
          handlers
          transport
-         
-      
+
+
       Migration Guides
       ----------------
-      
+
       See the guide below for instructions on migrating between major releases of this library.
-      
+
       .. toctree::
           :maxdepth: 2
-      
+
           UPGRADING
     count: 1
   - paths: [
-      "packages/google-cloud-logging/google/cloud/logging/__init__.py",      
+      "packages/google-cloud-logging/google/cloud/logging/__init__.py",
     ]
     before: |
       from google.cloud.logging import gapic_version as package_version
@@ -463,6 +463,24 @@ replacements:
           "django",
       ]
     count: 1
+  - paths: [
+      packages/google-cloud-logging/noxfile.py
+    ]
+    before: |
+      CURRENT_DIRECTORY = pathlib.Path\(__file__\).parent.absolute\(\)
+    after: |
+      CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+      # Path to the centralized mypy configuration file at the repository root.
+      # Search upwards to support running nox from both monorepo packages and integration test goldens.
+      MYPY_CONFIG_FILE = next(
+          (
+              str(p / "mypy.ini")
+              for p in CURRENT_DIRECTORY.parents
+              if (p / "mypy.ini").exists()
+          ),
+          str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
+      )
+    count: 1
   - paths: [
       packages/google-cloud-logging/noxfile.py
     ]
@@ -478,4 +496,19 @@ replacements:
           # TODO(https://github.com/googleapis/google-cloud-python/issues/13362):
           # Enable mypy once this repo has been updated for mypy evaluation.
           session.skip("Skip mypy since this library is not yet updated for mypy evaluation")
+
+          session.install("-e", ".")
+          session.install(
+              "mypy",
+              "types-setuptools",
+              "types-protobuf",
+              "types-requests",
+          )
+          session.run(
+              "mypy",
+              f"--config-file={MYPY_CONFIG_FILE}",
+              "-p",
+              "google",
+              *session.posargs,
+          )
     count: 1

.librarian/generator-input/client-post-processing/pubsub-integration.yaml

@@ -1,5 +1,5 @@
 replacements:
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/**/*client.py"
     before: 'instead\.\n(?:[ \t]*\n)*([ \t]+)DEFAULT_ENDPOINT = "pubsub\.googleapis\.com"'
     after: |-
@@ -13,7 +13,7 @@ replacements:
       \g<1>DEFAULT_ENDPOINT = "pubsub.googleapis.com"
     count: 3
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/*/client.py"
     before: 'import json\n(?:[ \t]*\n)*import logging as std_logging'
     after: |-
@@ -23,7 +23,7 @@ replacements:
       import logging as std_logging
     count: 3
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/*/client.py"
     before: 'class\n(?:[ \t]*\n)*([ \t]+)self\._transport = transport_init\('
     after: |-
@@ -38,7 +38,7 @@ replacements:
       \g<1>self._transport = transport_init(
     count: 3
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/*/transports/grpc*.py"
       - "packages/google-cloud-pubsub/tests/unit/gapic/pubsub_v1/*.py"
     before: '\("grpc\.max_receive_message_length", -1\),\n(?:[ \t]*\n)*([ \t]+)\]'
@@ -49,19 +49,19 @@ replacements:
       \g<1>]
     count: 21
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/publisher/transports/base.py"
     before: 'deadline=60\.0,'
     after: 'deadline=600.0,'
     count: 9
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/**/*.py"
     before: 'gapic_version=package_version\.__version__'
     after: 'client_library_version=package_version.__version__'
     count: 9
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
     before: 'import logging as std_logging\n(?:[ \t]*\n)*import os'
     after: |-
@@ -71,7 +71,7 @@ replacements:
       import os
     count: 1
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/async_client.py"
     before: 'import logging as std_logging\n(?:[ \t]*\n)*import re'
     after: |-
@@ -80,7 +80,7 @@ replacements:
       import re
     count: 1
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/async_client.py"
     before: 'request\.max_messages = max_messages\n(?:[ \t]*\n)*([ \t]+)# Wrap the RPC method'
@@ -96,7 +96,7 @@ replacements:
       \g<1># Wrap the RPC method
     count: 2
 
-  - paths: 
+  - paths:
       - "packages/google-cloud-pubsub/google/pubsub_v1/services/subscriber/client.py"
     before: '"""\n(?:[ \t]*\n)*([ \t]+)# Wrap the RPC method; this adds retry and timeout information,\n(?:[ \t]*\n)*[ \t]+# and friendly error handling\.\n(?:[ \t]*\n)*[ \t]+rpc = self\._transport\._wrapped_methods\[self\._transport\.streaming_pull\]'
     after: |-
@@ -403,6 +403,16 @@ replacements:
     before: 'nox\.options\.sessions = \[\n(?:[ \t]+)"unit",\n(?:[ \t]+)"system",\n(?:[ \t]+)"cover",\n(?:[ \t]+)"lint",\n(?:[ \t]+)"lint_setup_py",\n(?:[ \t]+)"blacken",\n(?:[ \t]+)"docs",\n\]'
     after: |-
       CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+      # Path to the centralized mypy configuration file at the repository root.
+      # Search upwards to support running nox from both monorepo packages and integration test goldens.
+      MYPY_CONFIG_FILE = next(
+          (
+              str(p / "mypy.ini")
+              for p in CURRENT_DIRECTORY.parents
+              if (p / "mypy.ini").exists()
+          ),
+          str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
+      )
 
       nox.options.sessions = [
           "unit",
@@ -450,7 +460,7 @@ replacements:
           # mypy checks yet.
           # https://github.com/googleapis/gapic-generator-python/issues/1092
           # TODO: Re-enable mypy checks once we merge, since incremental checks are failing due to protobuf upgrade
-          # session.run("mypy", "-p", "google.cloud", "--exclude", "google/pubsub_v1/")
+          # session.run("mypy", f"--config-file={MYPY_CONFIG_FILE}", "-p", "google.cloud", "--exclude", "google/pubsub_v1/")
 
 
       @nox.session(python=DEFAULT_PYTHON_VERSION)

.librarian/generator-input/client-post-processing/storage-integration.yaml

@@ -1,5 +1,23 @@
 description: Restore handwritten __init__.py for google-cloud-storage
 replacements:
+  - paths: [
+      packages/google-cloud-storage/noxfile.py,
+    ]
+    before: |
+      CURRENT_DIRECTORY = pathlib.Path\(__file__\).parent.absolute\(\)
+    after: |
+      CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+      # Path to the centralized mypy configuration file at the repository root.
+      # Search upwards to support running nox from both monorepo packages and integration test goldens.
+      MYPY_CONFIG_FILE = next(
+          (
+              str(p / "mypy.ini")
+              for p in CURRENT_DIRECTORY.parents
+              if (p / "mypy.ini").exists()
+          ),
+          str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
+      )
+    count: 1
   - paths: [
       packages/google-cloud-storage/noxfile.py,
     ]
@@ -804,4 +822,19 @@ replacements:
           # TODO(https://github.com/googleapis/google-cloud-python/issues/13362):
           # Enable mypy once this repo has been updated for mypy evaluation.
           session.skip("Skip mypy since this library is not yet updated for mypy evaluation")
+
+          session.install("-e", ".")
+          session.install(
+              "mypy",
+              "types-setuptools",
+              "types-protobuf",
+              "types-requests",
+          )
+          session.run(
+              "mypy",
+              f"--config-file={MYPY_CONFIG_FILE}",
+              "-p",
+              "google",
+              *session.posargs,
+          )
     count: 1

mypy.ini

@@ -27,31 +27,31 @@ ignore_missing_imports = True
 [mypy-flask]
 ignore_missing_imports = True
 
-[mypy-google.api.*]
+[mypy-google.api,google.api.*]
 ignore_missing_imports = True
 
 [mypy-google.colab]
 ignore_missing_imports = True
 
-[mypy-google.iam.*]
+[mypy-google.iam,google.iam.*]
 ignore_missing_imports = True
 
-[mypy-google.longrunning.*]
+[mypy-google.longrunning,google.longrunning.*]
 ignore_missing_imports = True
 
-[mypy-google.oauth2.*]
+[mypy-google.oauth2,google.oauth2.*]
 ignore_missing_imports = True
 
-[mypy-google.protobuf.*]
+[mypy-google.protobuf,google.protobuf.*]
 ignore_missing_imports = True
 
-[mypy-google.rpc.*]
+[mypy-google.rpc,google.rpc.*]
 ignore_missing_imports = True
 
-[mypy-google.type.*]
+[mypy-google.type,google.type.*]
 ignore_missing_imports = True
 
-[mypy-grpc.*]
+[mypy-grpc,grpc.*]
 ignore_missing_imports = True
 
 [mypy-grpc_status]
@@ -63,7 +63,7 @@ ignore_missing_imports = True
 [mypy-ipywidgets]
 ignore_missing_imports = True
 
-[mypy-proto.*]
+[mypy-proto,proto.*]
 ignore_missing_imports = True
 
 [mypy-pyarrow.*]
@@ -95,9 +95,11 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 
 [mypy-google.cloud.bigtable.*]
+# Broadly ignore errors for the older, legacy/handwritten parts of the library
 ignore_errors = True
 
 [mypy-google.cloud.bigtable.data.*]
+# Stricter rules for the newer parts of the library
 check_untyped_defs = True
 warn_unreachable = True
 disallow_any_generics = True

packages/bigquery-magics/mypy.ini

@@ -112,6 +112,17 @@
 }
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+# Path to the centralized mypy configuration file at the repository root.
+# Search upwards to support running nox from both monorepo packages and integration test goldens.
+MYPY_CONFIG_FILE = next(
+    (
+        str(p / "mypy.ini")
+        for p in CURRENT_DIRECTORY.parents
+        if (p / "mypy.ini").exists()
+    ),
+    str(CURRENT_DIRECTORY.parent.parent / "mypy.ini"),
+)
+
 
 nox.options.sessions = [
     "unit",
@@ -546,3 +557,18 @@ def mypy(session):
     # TODO(https://github.com/googleapis/google-cloud-python/issues/16014):
     # Add mypy tests
     session.skip("mypy tests are not yet supported")
+
+    session.install("-e", ".")
+    session.install(
+        "mypy",
+        "types-setuptools",
+        "types-protobuf",
+        "types-requests",
+    )
+    session.run(
+        "mypy",
+        f"--config-file={MYPY_CONFIG_FILE}",
+        "-p",
+        "bigquery_magics",
+        *session.posargs,
+    )