feat(clients): allow non-dev API endpoint roots

Specifying an API endpoint root to a client used to imply that the
endpoint in use was for dev, which would disable TLS and token bearer
authorisation. This is not always a valid assumption, for example when
manually specifying a locational endpoint for Google PubSub to target a
specific region, as such endpoints are for production and should
therefore use TLS and authorisation.

Fix this by allowing manual configuration of the `api_is_dev` setting
when using a non-dev root, whilst maintaining the old behaviour by
default for backwards compatibility.

Author: w-miller

auth/poetry.rest.lock

@@ -13,13 +13,13 @@ files = [
 
 [[package]]
 name = "certifi"
-version = "2024.6.2"
+version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
-    {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
+    {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+    {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
 ]
 
 [[package]]
@@ -353,21 +353,21 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
 
 [[package]]
 name = "pytest"
-version = "8.2.2"
+version = "8.3.3"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
-    {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
+    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
+    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
 ]
 
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.5,<2.0"
+pluggy = ">=1.5,<2"
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
@@ -424,13 +424,13 @@ files = [
 
 [[package]]
 name = "urllib3"
-version = "2.2.1"
+version = "2.2.2"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"},
-    {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"},
+    {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
+    {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},
 ]
 
 [package.extras]
@@ -442,4 +442,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">= 3.8, < 4.0"
-content-hash = "0a8e3c7f35afc47adf19cfc80379283f411511be97c5400c593f1e9d94ec1334"
+content-hash = "33d10c6c4fa764ec12e79a7cb93f22b67a92e4e174d364b1b47ad1bb411518ca"

auth/pyproject.rest.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gcloud-rest-auth"
-version = "5.3.1"
+version = "5.3.2"
 description = "Python Client for Google Cloud Auth"
 readme = "README.rst"
 
@@ -30,8 +30,8 @@ pyjwt = ">= 1.5.3, < 3.0.0"
 requests = ">= 2.2.1, < 3.0.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "8.2.2"
-# pytest-asyncio = "0.23.7"
+pytest = "8.3.3"
+# pytest-asyncio = "0.23.8"
 pytest-mock = "3.14.0"
 
 [[tool.poetry.source]]

bigquery/gcloud/aio/bigquery/bigquery.py

@@ -29,9 +29,14 @@
 log = logging.getLogger(__name__)
 
 
-def init_api_root(api_root: Optional[str]) -> Tuple[bool, str]:
+def init_api_root(
+        api_root: Optional[str], api_is_dev: Optional[bool],
+) -> Tuple[bool, str]:
     if api_root:
-        return True, api_root
+        if api_is_dev is None:
+            # Assume a provided API root is dev unless otherwise specified
+            api_is_dev = True
+        return api_is_dev, api_root
 
     host = os.environ.get('BIGQUERY_EMULATOR_HOST')
     if host:
@@ -69,9 +74,9 @@ def __init__(
             self, project: Optional[str] = None,
             service_file: Optional[Union[str, IO[AnyStr]]] = None,
             session: Optional[Session] = None, token: Optional[Token] = None,
-            api_root: Optional[str] = None,
+            api_root: Optional[str] = None, api_is_dev: Optional[bool] = None,
     ) -> None:
-        self._api_is_dev, self._api_root = init_api_root(api_root)
+        self._api_is_dev, self._api_root = init_api_root(api_root, api_is_dev)
         self.session = AioSession(session)
         self.token = token or Token(
             service_file=service_file, scopes=SCOPES,

bigquery/poetry.rest.lock

@@ -13,13 +13,13 @@ files = [
 
 [[package]]
 name = "certifi"
-version = "2024.6.2"
+version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
-    {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
+    {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+    {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
 ]
 
 [[package]]
@@ -277,7 +277,7 @@ test = ["pytest (>=6)"]
 
 [[package]]
 name = "gcloud-rest-auth"
-version = "5.3.1"
+version = "5.3.2"
 description = "Python Client for Google Cloud Auth"
 optional = false
 python-versions = ">= 3.8, < 4.0"
@@ -322,7 +322,7 @@ develop = false
 
 [package.dependencies]
 gcloud-rest-auth = ">= 5.3.0, < 6.0.0"
-pyasn1-modules = ">= 0.2.1, < 0.4.1"
+pyasn1-modules = ">= 0.2.1, < 0.4.2"
 rsa = ">= 3.1.4, < 5.0.0"
 
 [package.source]
@@ -432,21 +432,21 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
 
 [[package]]
 name = "pytest"
-version = "8.2.2"
+version = "8.3.3"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
-    {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
+    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
+    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
 ]
 
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.5,<2.0"
+pluggy = ">=1.5,<2"
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
@@ -517,13 +517,13 @@ files = [
 
 [[package]]
 name = "urllib3"
-version = "2.2.1"
+version = "2.2.2"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"},
-    {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"},
+    {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
+    {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},
 ]
 
 [package.extras]
@@ -535,4 +535,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">= 3.8, < 4.0"
-content-hash = "2b4c5e685ccbd1981c6304fba0d224993f96fd4e3e0cdc86327c06585f0edaa5"
+content-hash = "9dd0c655978fdba6dd4fa3f494140b57745f3af95747b85e0128720734150d49"

bigquery/pyproject.rest.toml

@@ -27,8 +27,8 @@ gcloud-rest-auth = ">= 3.1.0, < 6.0.0"
 gcloud-rest-auth = { path = "../auth" }
 gcloud-rest-datastore = { path = "../datastore" }
 gcloud-rest-storage = { path = "../storage" }
-pytest = "8.2.2"
-# pytest-asyncio = "0.23.7"
+pytest = "8.3.3"
+# pytest-asyncio = "0.23.8"
 pytest-mock = "3.14.0"
 
 [[tool.poetry.source]]

bigquery/tests/unit/bigquery_test.py

@@ -1,4 +1,8 @@
+from unittest.mock import AsyncMock
+
+import pytest
 from gcloud.aio.bigquery import Table
+from gcloud.aio.bigquery.bigquery import BigqueryBase
 
 
 def test_make_insert_body():
@@ -52,3 +56,34 @@ def test_make_insert_body_defult_id_fn():
 
     assert len(body['rows']) == 2
     assert all(r['insertId'] for r in body['rows'])
+
+
+# =========
+# client
+# =========
+@pytest.mark.asyncio
+async def test_client_api_is_dev():
+    """
+    Test that the api_is_dev constructor parameter controls whether the
+    Authorization header is set on requests
+    """
+    api_root = 'https://foobar/v1'
+
+    # With no API root specified, assume API not dev, so auth header should be
+    # set
+    async with BigqueryBase(token=AsyncMock()) as bigquery:
+        assert 'Authorization' in await bigquery.headers()
+    # If API root set and not otherwise specified, assume API is dev, so auth
+    # header should not be set
+    async with BigqueryBase(api_root=api_root, token=AsyncMock()) as bigquery:
+        assert 'Authorization' not in await bigquery.headers()
+    # If API specified to be dev, auth header should not be set
+    async with BigqueryBase(
+            api_root=api_root, api_is_dev=True, token=AsyncMock(),
+    ) as bigquery:
+        assert 'Authorization' not in await bigquery.headers()
+    # If API specified to not be dev, auth header should be set
+    async with BigqueryBase(
+            api_root=api_root, api_is_dev=False, token=AsyncMock(),
+    ) as bigquery:
+        assert 'Authorization' in await bigquery.headers()

datastore/gcloud/aio/datastore/datastore.py

@@ -44,9 +44,14 @@
 LookUpResult = Dict[str, Union[str, List[Union[EntityResult, Key]]]]
 
 
-def init_api_root(api_root: Optional[str]) -> Tuple[bool, str]:
+def init_api_root(
+        api_root: Optional[str], api_is_dev: Optional[bool],
+) -> Tuple[bool, str]:
     if api_root:
-        return True, api_root
+        if api_is_dev is None:
+            # Assume a provided API root is dev unless otherwise specified
+            api_is_dev = True
+        return api_is_dev, api_root
 
     host = os.environ.get('DATASTORE_EMULATOR_HOST')
     if host:
@@ -72,8 +77,9 @@ def __init__(
             service_file: Optional[Union[str, IO[AnyStr]]] = None,
             namespace: str = '', session: Optional[Session] = None,
             token: Optional[Token] = None, api_root: Optional[str] = None,
+            api_is_dev: Optional[bool] = None,
     ) -> None:
-        self._api_is_dev, self._api_root = init_api_root(api_root)
+        self._api_is_dev, self._api_root = init_api_root(api_root, api_is_dev)
         self.namespace = namespace
         self.session = AioSession(session)
         self.token = token or Token(

datastore/poetry.rest.lock

@@ -13,13 +13,13 @@ files = [
 
 [[package]]
 name = "certifi"
-version = "2024.6.2"
+version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
-    {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
+    {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+    {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
 ]
 
 [[package]]
@@ -277,7 +277,7 @@ test = ["pytest (>=6)"]
 
 [[package]]
 name = "gcloud-rest-auth"
-version = "5.3.1"
+version = "5.3.2"
 description = "Python Client for Google Cloud Auth"
 optional = false
 python-versions = ">= 3.8, < 4.0"
@@ -306,7 +306,7 @@ develop = false
 
 [package.dependencies]
 gcloud-rest-auth = ">= 5.3.0, < 6.0.0"
-pyasn1-modules = ">= 0.2.1, < 0.4.1"
+pyasn1-modules = ">= 0.2.1, < 0.4.2"
 rsa = ">= 3.1.4, < 5.0.0"
 
 [package.source]
@@ -416,21 +416,21 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
 
 [[package]]
 name = "pytest"
-version = "8.2.2"
+version = "8.3.3"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
-    {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
+    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
+    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
 ]
 
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.5,<2.0"
+pluggy = ">=1.5,<2"
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
@@ -501,13 +501,13 @@ files = [
 
 [[package]]
 name = "urllib3"
-version = "2.2.1"
+version = "2.2.2"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"},
-    {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"},
+    {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
+    {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},
 ]
 
 [package.extras]
@@ -519,4 +519,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">= 3.8, < 4.0"
-content-hash = "2fbf77bdf723122dd0065a96d7a0e62515a7fba608415504d434f0096d3aa081"
+content-hash = "09644a57b223ca0ce1aad4d82ef50b4544c5424031798809458ae5866f9edd92"

datastore/pyproject.rest.toml

@@ -27,8 +27,8 @@ gcloud-rest-auth = ">= 3.1.0, < 6.0.0"
 # aiohttp = "3.9.1"
 gcloud-rest-auth = { path = "../auth" }
 gcloud-rest-storage = { path = "../storage" }
-pytest = "8.2.2"
-# pytest-asyncio = "0.23.7"
+pytest = "8.3.3"
+# pytest-asyncio = "0.23.8"
 pytest-mock = "3.14.0"
 
 [[tool.poetry.source]]