fix: Concatenate paths in _get_base_url_with_base_path rather than use urljoin (#2448)

Author: erlendvollset

cognite/client/_api/files.py

@@ -8,7 +8,7 @@
 from io import BufferedReader
 from pathlib import Path
 from typing import Any, BinaryIO, Literal, TextIO, cast, overload
-from urllib.parse import urljoin, urlparse
+from urllib.parse import urlparse
 
 from cognite.client._api_client import APIClient
 from cognite.client._constants import _RUNNING_IN_BROWSER, DEFAULT_LIMIT_READ
@@ -28,7 +28,7 @@
 )
 from cognite.client.data_classes.data_modeling import NodeId
 from cognite.client.exceptions import CogniteAPIError, CogniteAuthorizationError, CogniteFileUploadError
-from cognite.client.utils._auxiliary import find_duplicates
+from cognite.client.utils._auxiliary import append_url_path, find_duplicates
 from cognite.client.utils._concurrency import execute_tasks
 from cognite.client.utils._identifier import Identifier, IdentifierSequence
 from cognite.client.utils._validation import process_asset_subtree_ids, process_data_set_ids
@@ -646,7 +646,7 @@ def _upload_bytes(self, content: bytes | TextIO | BinaryIO, returned_file_metada
         if urlparse(upload_url).netloc:
             full_upload_url = upload_url
         else:
-            full_upload_url = urljoin(self._config.base_url, upload_url)
+            full_upload_url = append_url_path(self._config.base_url, upload_url)
         file_metadata = FileMetadata._load(returned_file_metadata)
         upload_response = self._http_client_with_retry.request(
             "PUT",

cognite/client/_api/org_apis/principals.py

@@ -3,11 +3,11 @@
 import warnings
 from collections.abc import Sequence
 from typing import overload
-from urllib.parse import urljoin
 
 from cognite.client._constants import DEFAULT_LIMIT_READ
 from cognite.client._org_client import OrgAPIClient
 from cognite.client.data_classes.principals import Principal, PrincipalList
+from cognite.client.utils._auxiliary import append_url_path
 from cognite.client.utils._identifier import PrincipalIdentifierSequence
 from cognite.client.utils.useful_types import SequenceNotStr
 
@@ -33,7 +33,7 @@ def me(self) -> Principal:
         if self._api_version:
             path = f"/api/{self._api_version}{path}"
 
-        full_url = urljoin(self._auth_url, path)
+        full_url = append_url_path(self._auth_url, path)
         headers = self._configure_headers(
             "application/json",
             additional_headers=self._config.headers.copy(),

cognite/client/_api_client.py

@@ -19,7 +19,6 @@
     cast,
     overload,
 )
-from urllib.parse import urljoin
 
 import requests.utils
 from requests import Response
@@ -45,6 +44,7 @@
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError, CogniteProjectAccessError
 from cognite.client.utils import _json
 from cognite.client.utils._auxiliary import (
+    append_url_path,
     get_current_sdk_version,
     get_user_agent,
     interpolate_and_url_encode,
@@ -301,7 +301,7 @@ def _get_base_url_with_base_path(self) -> str:
         base_path = ""
         if self._api_version:
             base_path = f"/api/{self._api_version}/projects/{self._config.project}"
-        return urljoin(self._config.base_url, base_path)
+        return append_url_path(self._config.base_url, base_path)
 
     def _is_retryable(self, method: str, path: str) -> bool:
         valid_methods = ["GET", "POST", "PUT", "DELETE", "PATCH"]

cognite/client/_org_client.py

@@ -2,10 +2,10 @@
 
 from abc import ABC
 from functools import cached_property
-from urllib.parse import urljoin
 
 from cognite.client._api_client import APIClient
 from cognite.client.exceptions import CogniteAPIError
+from cognite.client.utils._auxiliary import append_url_path
 
 
 class OrgAPIClient(APIClient, ABC):
@@ -18,7 +18,7 @@ def _get_base_url_with_base_path(self) -> str:
             base_path = f"/api/{self._api_version}/orgs/{self._organization}"
         # The OrganizationAPi uses the auth_url as the base for these endpoints instead of the
         # base_url like the rest of the SDK.
-        return urljoin(self._auth_url, base_path)
+        return append_url_path(self._auth_url, base_path)
 
     @cached_property
     def _organization(self) -> str:
@@ -28,7 +28,7 @@ def _organization(self) -> str:
             api_subversion=self._api_subversion,
         )
         # This is an internal endpoint, not part of the public API
-        full_url = urljoin(self._config.base_url, f"/api/v1/projects/{self._config.project}")
+        full_url = append_url_path(self._config.base_url, f"/api/v1/projects/{self._config.project}")
         response = self._http_client_with_retry.request(method="GET", url=full_url, headers=headers)
         if response.status_code != 200:
             raise CogniteAPIError(

cognite/client/utils/_auxiliary.py

@@ -14,7 +14,7 @@
     TypeVar,
     overload,
 )
-from urllib.parse import quote
+from urllib.parse import quote, urlparse, urlunparse
 
 from cognite.client.utils import _json
 from cognite.client.utils._importing import local_import
@@ -277,3 +277,9 @@ def flatten_dict(d: dict[str, Any], parent_keys: tuple[str, ...], sep: str = "."
         else:
             items.append((sep.join((*parent_keys, key)), value))
     return dict(items)
+
+
+def append_url_path(base_url: str, path: str) -> str:
+    parsed = urlparse(base_url)
+    new_path = f"{parsed.path.rstrip('/')}/{path.lstrip('/')}"
+    return urlunparse(parsed._replace(path=new_path)).rstrip("/")

tests/tests_unit/test_api_client.py

@@ -199,6 +199,21 @@ def test__do_request_raises_unmodified_exception(self, api_client_with_token):
         exc_msg = exc_info.value.args[0]
         assert "contain NaN(s) or +/- Inf!" not in exc_msg
 
+    def test_client_with_base_url_including_path_segments(self, cognite_client):
+        client = APIClient(
+            ClientConfig(
+                client_name="any",
+                project="test-project",
+                base_url="https://bla.bla.com/extra",
+                max_workers=1,
+                headers={"x-cdp-app": "python-sdk-integration-tests"},
+                credentials=Token(lambda: "abc"),
+            ),
+            api_version="v1",
+            cognite_client=cognite_client,
+        )
+        assert client._get_base_url_with_base_path() == "https://bla.bla.com/extra/api/v1/projects/test-project"
+
 
 class SomeUpdate(CogniteUpdate):
     @property