fix(auth): use requests transport for GCE MDS (#16480)

Fixes #16090

We were seeing errors on GCE environments, because the library would use
_http_client to communicate to the metadata server, even when mTLS is
enabled. This resulted in failures, because _http_client doesn't support
https.

This PR addresses the issue by using requests as the transport instead.
This depends on the optional `requests` dependency, but requests is
already [required by compute
engine](https://github.com/googleapis/google-cloud-python/blob/51a96200324965f071dd9542d25b907c1420b3f6/packages/google-auth/google/auth/compute_engine/_metadata.py#L27),
so there are no changes there.

I was able to reproduce and verify this test on a GCE VM

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: daniel-sanche

packages/google-auth/google/auth/_default.py

@@ -27,11 +27,10 @@
 
 from google.auth import environment_vars
 from google.auth import exceptions
-import google.auth.transport._http_client
 
 if TYPE_CHECKING:  # pragma: NO COVER
-    from google.auth.credentials import Credentials  # noqa: F401
-    from google.auth.transport import Request  # noqa: F401
+    import google.auth.credentials.Credentials  # type: ignore
+    import google.auth.transport.Request  # type: ignore
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -390,22 +389,19 @@ def _get_gae_credentials():
 
 def _get_gce_credentials(request=None, quota_project_id=None):
     """Gets credentials and project ID from the GCE Metadata Service."""
-    # Ping requires a transport, but we want application default credentials
-    # to require no arguments. So, we'll use the _http_client transport which
-    # uses http.client. This is only acceptable because the metadata server
-    # doesn't do SSL and never requires proxies.
-
     # While this library is normally bundled with compute_engine, there are
     # some cases where it's not available, so we tolerate ImportError.
+    # Compute Engine requires optional `requests` dependency.
     try:
         from google.auth import compute_engine
         from google.auth.compute_engine import _metadata
+        import google.auth.transport.requests
     except ImportError:
         _LOGGER.warning("Import of Compute Engine auth library failed.")
         return None, None
 
     if request is None:
-        request = google.auth.transport._http_client.Request()
+        request = google.auth.transport.requests.Request()
 
     if _metadata.is_on_gce(request=request):
         # Get the project ID.

packages/google-auth/google/auth/transport/_http_client.py

@@ -94,7 +94,7 @@ def __call__(
         if parts.scheme != "http":
             raise exceptions.TransportError(
                 "http.client transport only supports the http scheme, {}"
-                "was specified".format(parts.scheme)
+                " was specified".format(parts.scheme)
             )
 
         connection = http_client.HTTPConnection(parts.netloc, timeout=timeout)

packages/google-auth/tests/test__default.py

@@ -890,6 +890,18 @@ def test__get_gce_credentials_explicit_request(ping):
     ping.assert_called_with(request=mock.sentinel.request)
 
 
+@mock.patch(
+    "google.auth.compute_engine._metadata.is_on_gce", return_value=False, autospec=True
+)
+@mock.patch("google.auth.transport.requests.Request", autospec=True)
+def test__get_gce_credentials_default_request(mock_request_cls, ping):
+    credentials, project_id = _default._get_gce_credentials()
+    mock_request_cls.assert_called_once()
+    ping.assert_called_with(request=mock_request_cls.return_value)
+    assert credentials is None
+    assert project_id is None
+
+
 @mock.patch(
     "google.auth._default._get_explicit_environ_credentials",
     return_value=(MOCK_CREDENTIALS, mock.sentinel.project_id),
@@ -1006,6 +1018,35 @@ def test_default_fail(unused_gce, unused_gae, unused_sdk, unused_explicit):
     assert excinfo.match(_default._CLOUD_SDK_MISSING_CREDENTIALS)
 
 
+@mock.patch(
+    "google.auth._default._get_explicit_environ_credentials",
+    return_value=(None, None),
+    autospec=True,
+)
+@mock.patch(
+    "google.auth._default._get_gcloud_sdk_credentials",
+    return_value=(None, None),
+    autospec=True,
+)
+@mock.patch(
+    "google.auth._default._get_gae_credentials",
+    return_value=(None, None),
+    autospec=True,
+)
+@mock.patch(
+    "google.auth.compute_engine._metadata.is_on_gce", return_value=False, autospec=True
+)
+@mock.patch("google.auth.transport.requests.Request", autospec=True)
+def test_default_gce_triggers_request_creation(
+    mock_request_cls, is_on_gce, unused_gae, unused_sdk, unused_explicit
+):
+    with pytest.raises(exceptions.DefaultCredentialsError):
+        _default.default()
+
+    mock_request_cls.assert_called_once()
+    is_on_gce.assert_called_once_with(request=mock_request_cls.return_value)
+
+
 @mock.patch(
     "google.auth._default._get_explicit_environ_credentials",
     return_value=(MOCK_CREDENTIALS, mock.sentinel.project_id),