Skip to content

Commit 630a343

Browse files
andersfyllingclaudehaakonvt
authored
feat(records): add delete endpoint with infrastructure scaffolding (#2631)
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Håkon V. Treider <haakonvt@gmail.com>
1 parent c7028f9 commit 630a343

13 files changed

Lines changed: 327 additions & 2 deletions

File tree

cognite/client/_api/data_modeling/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from cognite.client._api.data_modeling.data_models import DataModelsAPI
88
from cognite.client._api.data_modeling.graphql import DataModelingGraphQLAPI
99
from cognite.client._api.data_modeling.instances import InstancesAPI
10+
from cognite.client._api.data_modeling.records import RecordsAPI
1011
from cognite.client._api.data_modeling.spaces import SpacesAPI
1112
from cognite.client._api.data_modeling.statistics import StatisticsAPI
1213
from cognite.client._api.data_modeling.streams import StreamsAPI
@@ -27,6 +28,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client
2728
self.views = ViewsAPI(config, api_version, cognite_client)
2829
self.instances = InstancesAPI(config, api_version, cognite_client)
2930
self.graphql = DataModelingGraphQLAPI(config, api_version, cognite_client)
31+
self.records = RecordsAPI(config, api_version, cognite_client)
3032
self.statistics = StatisticsAPI(config, api_version, cognite_client)
3133
self.streams = StreamsAPI(config, api_version, cognite_client)
3234

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
from __future__ import annotations
2+
3+
import asyncio
4+
from collections.abc import Sequence
5+
from typing import TYPE_CHECKING, ClassVar, Literal
6+
7+
from cognite.client._api_client import APIClient
8+
from cognite.client.data_classes.data_modeling.records import RecordId, RecordIdSequence
9+
from cognite.client.utils._concurrency import RecordsConcurrencyOperation
10+
from cognite.client.utils._experimental import FeaturePreviewWarning
11+
from cognite.client.utils._url import interpolate_and_url_encode
12+
13+
if TYPE_CHECKING:
14+
from cognite.client import AsyncCogniteClient
15+
from cognite.client.config import ClientConfig
16+
17+
18+
class RecordsAPI(APIClient):
19+
def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None:
20+
super().__init__(config, api_version, cognite_client)
21+
self._warning = FeaturePreviewWarning(
22+
api_maturity="General Availability", sdk_maturity="alpha", feature_name="Records"
23+
)
24+
25+
_OPERATION_TO_RATE_LIMIT: ClassVar[dict[str, RecordsConcurrencyOperation]] = {
26+
"write": RecordsConcurrencyOperation.WRITE,
27+
"delete": RecordsConcurrencyOperation.WRITE,
28+
}
29+
30+
def _get_semaphore(self, operation: Literal["write", "delete"]) -> asyncio.BoundedSemaphore:
31+
from cognite.client import global_config
32+
33+
return global_config.concurrency_settings.records._semaphore_factory(
34+
self._OPERATION_TO_RATE_LIMIT[operation], project=self._cognite_client.config.project
35+
)
36+
37+
def _records_url(self, stream_id: str, suffix: str = "") -> str:
38+
return interpolate_and_url_encode("/streams/{}/records{}", stream_id, suffix)
39+
40+
async def delete(
41+
self,
42+
items: RecordId | Sequence[RecordId],
43+
*,
44+
stream_id: str,
45+
ignore_unknown_ids: Literal[True] = True,
46+
) -> None:
47+
"""`Delete records from a stream <https://api-docs.cognite.com/20230101/tag/Records/operation/deleteRecords>`_.
48+
49+
Only valid for mutable streams (returns 422 on immutable). Unknown
50+
``space + externalId`` pairs are silently ignored.
51+
52+
Args:
53+
items (RecordId | Sequence[RecordId]): Records to delete.
54+
stream_id (str): External ID of the stream to delete from.
55+
ignore_unknown_ids (Literal[True]): Currently only True is supported
56+
57+
Examples:
58+
59+
Delete records:
60+
61+
>>> from cognite.client import CogniteClient
62+
>>> from cognite.client.data_classes.data_modeling.records import RecordId
63+
>>> client = CogniteClient()
64+
>>> client.data_modeling.records.delete(
65+
... stream_id="my-stream",
66+
... items=[
67+
... RecordId(space="my-space", external_id="rec-1"),
68+
... RecordId(space="my-space", external_id="rec-2"),
69+
... ],
70+
... )
71+
"""
72+
self._warning.warn()
73+
await self._delete_multiple(
74+
identifiers=RecordIdSequence.load(items),
75+
wrap_ids=True,
76+
resource_path=self._records_url(stream_id),
77+
)

cognite/client/_cognite_client.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
from cognite.client._api.data_modeling.data_models import DataModelsAPI
5050
from cognite.client._api.data_modeling.graphql import DataModelingGraphQLAPI
5151
from cognite.client._api.data_modeling.instances import InstancesAPI
52+
from cognite.client._api.data_modeling.records import RecordsAPI
5253
from cognite.client._api.data_modeling.space_statistics import SpaceStatisticsAPI
5354
from cognite.client._api.data_modeling.spaces import SpacesAPI
5455
from cognite.client._api.data_modeling.statistics import StatisticsAPI
@@ -439,6 +440,7 @@ def _make_accessors_for_building_docs() -> None:
439440
AsyncCogniteClient.data_modeling.statistics = StatisticsAPI # type: ignore
440441
AsyncCogniteClient.data_modeling.statistics.spaces = SpaceStatisticsAPI # type: ignore
441442
AsyncCogniteClient.data_modeling.streams = StreamsAPI # type: ignore
443+
AsyncCogniteClient.data_modeling.records = RecordsAPI # type: ignore
442444
AsyncCogniteClient.documents = DocumentsAPI # type: ignore
443445
AsyncCogniteClient.documents.previews = DocumentPreviewAPI # type: ignore
444446
AsyncCogniteClient.workflows = WorkflowAPI # type: ignore

cognite/client/_sync_api/data_modeling/__init__.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,28 @@
11
"""
22
===============================================================================
3-
584030bc5e2a4b8168f54c101f7f521d
3+
b80632521fdada43a35c314cc92b055b
44
This file is auto-generated from the Async API modules, - do not edit manually!
55
===============================================================================
66
"""
77

88
from __future__ import annotations
99

10-
from cognite.client import AsyncCogniteClient
10+
from typing import TYPE_CHECKING
11+
1112
from cognite.client._sync_api.data_modeling.containers import SyncContainersAPI
1213
from cognite.client._sync_api.data_modeling.data_models import SyncDataModelsAPI
1314
from cognite.client._sync_api.data_modeling.graphql import SyncDataModelingGraphQLAPI
1415
from cognite.client._sync_api.data_modeling.instances import SyncInstancesAPI
16+
from cognite.client._sync_api.data_modeling.records import SyncRecordsAPI
1517
from cognite.client._sync_api.data_modeling.spaces import SyncSpacesAPI
1618
from cognite.client._sync_api.data_modeling.statistics import SyncStatisticsAPI
1719
from cognite.client._sync_api.data_modeling.streams import SyncStreamsAPI
1820
from cognite.client._sync_api.data_modeling.views import SyncViewsAPI
1921
from cognite.client._sync_api_client import SyncAPIClient
2022

23+
if TYPE_CHECKING:
24+
from cognite.client import AsyncCogniteClient
25+
2126

2227
class SyncDataModelingAPI(SyncAPIClient):
2328
"""Auto-generated, do not modify manually."""
@@ -30,5 +35,6 @@ def __init__(self, async_client: AsyncCogniteClient) -> None:
3035
self.views = SyncViewsAPI(async_client)
3136
self.instances = SyncInstancesAPI(async_client)
3237
self.graphql = SyncDataModelingGraphQLAPI(async_client)
38+
self.records = SyncRecordsAPI(async_client)
3339
self.statistics = SyncStatisticsAPI(async_client)
3440
self.streams = SyncStreamsAPI(async_client)
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
"""
2+
===============================================================================
3+
d79de505c1799cf6ccaa471a4cd6701b
4+
This file is auto-generated from the Async API modules, - do not edit manually!
5+
===============================================================================
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from collections.abc import Sequence
11+
from typing import TYPE_CHECKING, Literal
12+
13+
from cognite.client import AsyncCogniteClient
14+
from cognite.client._sync_api_client import SyncAPIClient
15+
from cognite.client.data_classes.data_modeling.records import RecordId
16+
from cognite.client.utils._async_helpers import run_sync
17+
18+
if TYPE_CHECKING:
19+
from cognite.client import AsyncCogniteClient
20+
21+
22+
class SyncRecordsAPI(SyncAPIClient):
23+
"""Auto-generated, do not modify manually."""
24+
25+
def __init__(self, async_client: AsyncCogniteClient) -> None:
26+
self.__async_client = async_client
27+
28+
def delete(
29+
self, items: RecordId | Sequence[RecordId], *, stream_id: str, ignore_unknown_ids: Literal[True] = True
30+
) -> None:
31+
"""
32+
`Delete records from a stream <https://api-docs.cognite.com/20230101/tag/Records/operation/deleteRecords>`_.
33+
34+
Only valid for mutable streams (returns 422 on immutable). Unknown
35+
``space + externalId`` pairs are silently ignored.
36+
37+
Args:
38+
items (RecordId | Sequence[RecordId]): Records to delete.
39+
stream_id (str): External ID of the stream to delete from.
40+
ignore_unknown_ids (Literal[True]): Currently only True is supported
41+
42+
Examples:
43+
44+
Delete records:
45+
46+
>>> from cognite.client import CogniteClient
47+
>>> from cognite.client.data_classes.data_modeling.records import RecordId
48+
>>> client = CogniteClient()
49+
>>> client.data_modeling.records.delete(
50+
... stream_id="my-stream",
51+
... items=[
52+
... RecordId(space="my-space", external_id="rec-1"),
53+
... RecordId(space="my-space", external_id="rec-2"),
54+
... ],
55+
... )
56+
"""
57+
return run_sync(
58+
self.__async_client.data_modeling.records.delete(
59+
items=items, stream_id=stream_id, ignore_unknown_ids=ignore_unknown_ids
60+
)
61+
)

cognite/client/data_classes/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
EntityMatchingPredictionResult,
3030
JobStatus,
3131
)
32+
from cognite.client.data_classes.data_modeling.records import RecordId
3233
from cognite.client.data_classes.data_sets import (
3334
DataSet,
3435
DataSetFilter,
@@ -456,6 +457,7 @@
456457
"LimitList",
457458
"OidcCredentials",
458459
"RawTable",
460+
"RecordId",
459461
"Relationship",
460462
"RelationshipFilter",
461463
"RelationshipList",

cognite/client/data_classes/data_modeling/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
Union,
120120
UnionAll,
121121
)
122+
from cognite.client.data_classes.data_modeling.records import RecordId
122123
from cognite.client.data_classes.data_modeling.spaces import Space, SpaceApply, SpaceApplyList, SpaceList
123124
from cognite.client.data_classes.data_modeling.streams import (
124125
Stream,
@@ -242,6 +243,7 @@
242243
"Query",
243244
"QueryResult",
244245
"QuerySync",
246+
"RecordId",
245247
"RequiresConstraint",
246248
"RequiresConstraintApply",
247249
"ResultSetExpression",
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from __future__ import annotations
2+
3+
from collections.abc import Sequence
4+
5+
from cognite.client.utils._identifier import IdentifierSequenceCore, RecordId
6+
7+
__all__ = ["RecordId", "RecordIdSequence"]
8+
9+
10+
class RecordIdSequence(IdentifierSequenceCore[RecordId]):
11+
@classmethod
12+
def load(cls, items: RecordId | Sequence[RecordId]) -> RecordIdSequence:
13+
if isinstance(items, RecordId):
14+
return cls([items], is_singleton=True)
15+
return cls(list(items), is_singleton=False)

cognite/client/testing.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from cognite.client._api.data_modeling.data_models import DataModelsAPI
1818
from cognite.client._api.data_modeling.graphql import DataModelingGraphQLAPI
1919
from cognite.client._api.data_modeling.instances import InstancesAPI
20+
from cognite.client._api.data_modeling.records import RecordsAPI
2021
from cognite.client._api.data_modeling.space_statistics import SpaceStatisticsAPI
2122
from cognite.client._api.data_modeling.spaces import SpacesAPI
2223
from cognite.client._api.data_modeling.statistics import StatisticsAPI
@@ -101,6 +102,7 @@
101102
from cognite.client._sync_api.data_modeling.data_models import SyncDataModelsAPI
102103
from cognite.client._sync_api.data_modeling.graphql import SyncDataModelingGraphQLAPI
103104
from cognite.client._sync_api.data_modeling.instances import SyncInstancesAPI
105+
from cognite.client._sync_api.data_modeling.records import SyncRecordsAPI
104106
from cognite.client._sync_api.data_modeling.space_statistics import SyncSpaceStatisticsAPI
105107
from cognite.client._sync_api.data_modeling.spaces import SyncSpacesAPI
106108
from cognite.client._sync_api.data_modeling.statistics import SyncStatisticsAPI
@@ -226,6 +228,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
226228
dm_views = create_autospec(ViewsAPI, instance=True, spec_set=True)
227229
dm_instances = create_autospec(InstancesAPI, instance=True, spec_set=True)
228230
dm_graphql = create_autospec(DataModelingGraphQLAPI, instance=True, spec_set=True)
231+
dm_records = create_autospec(RecordsAPI, instance=True, spec_set=True)
229232
dm_streams = create_autospec(StreamsAPI, instance=True, spec_set=True)
230233
self.data_modeling = create_autospec(
231234
DataModelingAPI,
@@ -237,6 +240,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
237240
instances=dm_instances,
238241
graphql=dm_graphql,
239242
statistics=dm_statistics,
243+
records=dm_records,
240244
streams=dm_streams,
241245
)
242246
flip_spec_set_on(self.data_modeling, dm_statistics)
@@ -427,6 +431,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
427431
dm_views = create_autospec(SyncViewsAPI, instance=True, spec_set=True)
428432
dm_instances = create_autospec(SyncInstancesAPI, instance=True, spec_set=True)
429433
dm_graphql = create_autospec(SyncDataModelingGraphQLAPI, instance=True, spec_set=True)
434+
dm_records = create_autospec(SyncRecordsAPI, instance=True, spec_set=True)
430435
dm_streams = create_autospec(SyncStreamsAPI, instance=True, spec_set=True)
431436
self.data_modeling = create_autospec(
432437
SyncDataModelingAPI,
@@ -438,6 +443,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
438443
instances=dm_instances,
439444
graphql=dm_graphql,
440445
statistics=dm_statistics,
446+
records=dm_records,
441447
streams=dm_streams,
442448
)
443449
flip_spec_set_on(self.data_modeling, dm_statistics)

cognite/client/utils/_concurrency.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from abc import ABC, abstractmethod
88
from collections import UserList
99
from collections.abc import Callable, Coroutine
10+
from enum import Enum
1011
from typing import (
1112
Any,
1213
Literal,
@@ -221,6 +222,47 @@ def __repr__(self) -> str:
221222
)
222223

223224

225+
class RecordsConcurrencyOperation(Enum):
226+
WRITE = "write"
227+
228+
229+
class RecordsGlobalConcurrencyConfig(ConcurrencyConfig):
230+
"""
231+
Global concurrency settings for the Records API. Named "global" to distinguish from
232+
future per-endpoint rate limits that may be added later.
233+
234+
Args:
235+
concurrency_settings (ConcurrencySettings): Reference to the parent settings object.
236+
write (int): Maximum concurrent write requests (ingest, delete).
237+
"""
238+
239+
def __init__(
240+
self,
241+
concurrency_settings: ConcurrencySettings,
242+
write: int,
243+
) -> None:
244+
super().__init__(concurrency_settings, "records", read=0, write=write, delete=0)
245+
246+
def _semaphore_factory(self, operation: RecordsConcurrencyOperation, project: str) -> asyncio.BoundedSemaphore:
247+
key = (operation.value, project, asyncio.get_running_loop())
248+
if key in self._semaphore_cache:
249+
return self._semaphore_cache[key]
250+
251+
from cognite.client import global_config
252+
253+
global_config.concurrency_settings._freeze()
254+
match operation:
255+
case RecordsConcurrencyOperation.WRITE:
256+
sem = asyncio.BoundedSemaphore(self._write)
257+
case _:
258+
assert_never(operation)
259+
self._semaphore_cache[key] = sem
260+
return sem
261+
262+
def __repr__(self) -> str:
263+
return f"Concurrency[records](write={self._write})"
264+
265+
224266
class FileConcurrencyConfig(ConcurrencyConfig):
225267
"""
226268
Concurrency settings for the Files API.
@@ -383,6 +425,7 @@ def __init__(self) -> None:
383425
write_schema=1,
384426
)
385427
self._files = FileConcurrencyConfig(self, read=4, write=2, upload=5, download=5, delete=2, open_files=15)
428+
self._records = RecordsGlobalConcurrencyConfig(self, write=20)
386429

387430
@functools.cached_property
388431
def _all_concurrency_configs(self) -> list[ConcurrencyConfig]:
@@ -428,6 +471,10 @@ def raw(self) -> CRUDConcurrency:
428471
def files(self) -> FileConcurrencyConfig:
429472
return self._files
430473

474+
@property
475+
def records(self) -> RecordsGlobalConcurrencyConfig:
476+
return self._records
477+
431478
def __repr__(self) -> str:
432479
frozen_str = " (frozen)" if self.__frozen else ""
433480
return (
@@ -437,6 +484,7 @@ def __repr__(self) -> str:
437484
f" datapoints={self._datapoints},\n"
438485
f" data_modeling={self._data_modeling},\n"
439486
f" files={self._files},\n"
487+
f" records={self._records},\n"
440488
f"){frozen_str}"
441489
)
442490

0 commit comments

Comments
 (0)