Skip to content

Commit fcd74e6

Browse files
haakonvthaakonvt
authored
feat(Data Modeling, Files): Add files api to data modeling with /retrieve and /list (#2647)
1 parent 39ef35b commit fcd74e6

13 files changed

Lines changed: 627 additions & 8 deletions

File tree

‎cognite/client/_api/data_modeling/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
from cognite.client._api.data_modeling.containers import ContainersAPI
77
from cognite.client._api.data_modeling.data_models import DataModelsAPI
8+
from cognite.client._api.data_modeling.files import DataModelingFilesAPI
89
from cognite.client._api.data_modeling.graphql import DataModelingGraphQLAPI
910
from cognite.client._api.data_modeling.instances import InstancesAPI
1011
from cognite.client._api.data_modeling.records import RecordsAPI
@@ -24,6 +25,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client
2425
super().__init__(config, api_version, cognite_client)
2526
self.containers = ContainersAPI(config, api_version, cognite_client)
2627
self.data_models = DataModelsAPI(config, api_version, cognite_client)
28+
self.files = DataModelingFilesAPI(config, api_version, cognite_client)
2729
self.spaces = SpacesAPI(config, api_version, cognite_client)
2830
self.views = ViewsAPI(config, api_version, cognite_client)
2931
self.instances = InstancesAPI(config, api_version, cognite_client)

‎cognite/client/_api/data_modeling/files.py‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
from __future__ import annotations
2+
3+
from collections.abc import Sequence
4+
from typing import TYPE_CHECKING, Any, NoReturn, overload
5+
6+
from cognite.client._api_client import APIClient
7+
from cognite.client._constants import DEFAULT_LIMIT_READ
8+
from cognite.client.data_classes.data_modeling.cdm.v1 import CogniteFile
9+
from cognite.client.data_classes.data_modeling.ids import NodeId, ViewId
10+
from cognite.client.data_classes.data_modeling.instances import InstanceSort, Node, NodeList
11+
from cognite.client.data_classes.data_modeling.views import View
12+
from cognite.client.data_classes.filters import Filter
13+
from cognite.client.utils.useful_types import SequenceNotStr
14+
15+
if TYPE_CHECKING:
16+
from cognite.client import AsyncCogniteClient
17+
from cognite.client.config import ClientConfig
18+
19+
COGNITE_FILE_VIEW_ID = CogniteFile.get_source()
20+
21+
22+
def _resolve_source(source: View | ViewId | tuple[str, str, str]) -> tuple[list[ViewId], bool]:
23+
match source:
24+
case ViewId():
25+
source_as_id = source
26+
case View():
27+
source_as_id = source.as_id()
28+
case [str(), str(), str()]:
29+
source_as_id = ViewId(*source)
30+
case _:
31+
raise TypeError(f"Expected View, ViewId, or a (space, external_id, version) tuple, got {type(source)}")
32+
33+
if source_as_id == COGNITE_FILE_VIEW_ID:
34+
return [source_as_id], False
35+
36+
# User has passed a custom source, we include CogniteFile source to guarantee only file nodes
37+
# are returned. We will later strip them (hence the 'True' flag) to avoid returning nodes with
38+
# properties from multiple sources as they are very annoying to work with in the SDK.
39+
return [source_as_id, COGNITE_FILE_VIEW_ID], True
40+
41+
42+
class DataModelingFilesAPI(APIClient):
43+
def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None:
44+
super().__init__(config, api_version, cognite_client)
45+
self._files_api = cognite_client.files
46+
47+
async def retrieve_download_urls(self, *args: Any, **kwargs: Any) -> NoReturn:
48+
raise NotImplementedError("This method is not implemented yet!")
49+
50+
async def download(self, *args: Any, **kwargs: Any) -> NoReturn:
51+
raise NotImplementedError("This method is not implemented yet!")
52+
53+
async def download_to_path(self, *args: Any, **kwargs: Any) -> NoReturn:
54+
raise NotImplementedError("This method is not implemented yet!")
55+
56+
async def download_bytes(self, *args: Any, **kwargs: Any) -> NoReturn:
57+
raise NotImplementedError("This method is not implemented yet!")
58+
59+
async def upload(self, *args: Any, **kwargs: Any) -> NoReturn:
60+
raise NotImplementedError("This method is not implemented yet!")
61+
62+
async def upload_bytes(self, *args: Any, **kwargs: Any) -> NoReturn:
63+
raise NotImplementedError("This method is not implemented yet!")
64+
65+
async def upload_content(self, *args: Any, **kwargs: Any) -> NoReturn:
66+
raise NotImplementedError("This method is not implemented yet!")
67+
68+
async def upload_content_bytes(self, *args: Any, **kwargs: Any) -> NoReturn:
69+
raise NotImplementedError("This method is not implemented yet!")
70+
71+
async def __call__(self) -> NoReturn:
72+
raise NotImplementedError("This method is not implemented yet!")
73+
74+
@overload
75+
async def retrieve(
76+
self,
77+
node_ids: NodeId | tuple[str, str],
78+
*,
79+
source: View | ViewId | tuple[str, str, str] = COGNITE_FILE_VIEW_ID,
80+
) -> Node | None: ...
81+
82+
@overload
83+
async def retrieve(
84+
self,
85+
node_ids: Sequence[NodeId] | Sequence[tuple[str, str]],
86+
*,
87+
source: View | ViewId | tuple[str, str, str] = COGNITE_FILE_VIEW_ID,
88+
) -> NodeList[Node]: ...
89+
90+
async def retrieve(
91+
self,
92+
node_ids: NodeId | tuple[str, str] | Sequence[NodeId] | Sequence[tuple[str, str]],
93+
*,
94+
source: View | ViewId | tuple[str, str, str] = COGNITE_FILE_VIEW_ID,
95+
) -> Node | NodeList[Node] | None:
96+
"""`Retrieve one or more files by instance ID. <https://api-docs.cognite.com/20230101/tag/Instances/operation/byExternalIdsInstances>`_
97+
98+
Only nodes that are files (i.e. have data in the CogniteFile view) will be returned.
99+
If a single instance ID is requested and it is not found, ``None`` is returned.
100+
101+
Args:
102+
node_ids (NodeId | tuple[str, str] | Sequence[NodeId] | Sequence[tuple[str, str]]): Single instance ID or a list of instance IDs.
103+
source (View | ViewId | tuple[str, str, str]): The view to fetch properties from. Defaults to CogniteFile.
104+
105+
Returns:
106+
Node | NodeList[Node] | None: A single ``Node`` (or ``None`` if not found) when given a single identifier, or a ``NodeList`` when given a sequence.
107+
108+
Examples:
109+
110+
Retrieve a single file by instance ID:
111+
112+
>>> from cognite.client import CogniteClient
113+
>>> from cognite.client.data_classes.data_modeling import NodeId
114+
>>> client = CogniteClient()
115+
>>> res = client.data_modeling.files.retrieve(NodeId("my-space", "my-file"))
116+
117+
Using a tuple shorthand:
118+
119+
>>> res = client.data_modeling.files.retrieve(("my-space", "my-file"))
120+
121+
Retrieve multiple file nodes:
122+
123+
>>> res = client.data_modeling.files.retrieve(
124+
... [("my-space", "file-1"), ("my-space", "file-2")]
125+
... )
126+
127+
Fetch properties from a custom view (note, only files will be returned):
128+
129+
>>> from cognite.client.data_classes.data_modeling import ViewId
130+
>>> res = client.data_modeling.files.retrieve(
131+
... NodeId("my-space", "my-file"),
132+
... source=ViewId("my-space", "MyFileExtension", "v1"),
133+
... )
134+
"""
135+
sources, strip = _resolve_source(source)
136+
result = await self._cognite_client.data_modeling.instances.retrieve_nodes(nodes=node_ids, sources=sources)
137+
if strip and result:
138+
for node in [result] if isinstance(result, Node) else result:
139+
node.drop_source(COGNITE_FILE_VIEW_ID)
140+
return result
141+
142+
async def list(
143+
self,
144+
*,
145+
source: View | ViewId | tuple[str, str, str] = COGNITE_FILE_VIEW_ID,
146+
space: str | SequenceNotStr[str] | None = None,
147+
sort: Sequence[InstanceSort | dict] | InstanceSort | dict | None = None,
148+
filter: Filter | dict[str, Any] | None = None,
149+
limit: int | None = DEFAULT_LIMIT_READ,
150+
) -> NodeList[Node]:
151+
"""`List file nodes. <https://api-docs.cognite.com/20230101/tag/Instances/operation/listInstances>`_
152+
153+
Only file nodes will be returned, regardless of the source passed.
154+
155+
Args:
156+
source (View | ViewId | tuple[str, str, str]): The view to fetch properties from. Defaults to CogniteFile.
157+
space (str | SequenceNotStr[str] | None): Restrict results to this space (or list of spaces).
158+
sort (Sequence[InstanceSort | dict] | InstanceSort | dict | None): Sort order for the results.
159+
filter (Filter | dict[str, Any] | None): Advanced filter to apply. See :class:`~cognite.client.data_classes.filters`.
160+
limit (int | None): Maximum number of results to return. Defaults to 25. Set to -1, float("inf") or None to return all items.
161+
162+
Returns:
163+
NodeList[Node]: The matching files.
164+
165+
Examples:
166+
167+
List a few files:
168+
169+
>>> from cognite.client import CogniteClient
170+
>>> client = CogniteClient()
171+
>>> res = client.data_modeling.files.list(limit=5)
172+
173+
List all files in a specific space:
174+
175+
>>> res = client.data_modeling.files.list(space="my-space", limit=None)
176+
177+
Fetch properties from a custom view (note, only files will be returned), and
178+
apply a custom filter on the file name:
179+
180+
>>> from cognite.client.data_classes.data_modeling import ViewId
181+
>>> from cognite.client.data_classes import filters
182+
>>> view_id = ViewId("my-space", "MyFileExtension", "v1")
183+
>>> res = client.data_modeling.files.list(
184+
... source=view_id,
185+
... filter=filters.Prefix(view_id.as_property_ref("name"), "report"),
186+
... limit=None,
187+
... )
188+
"""
189+
sources, strip = _resolve_source(source)
190+
results = await self._cognite_client.data_modeling.instances.list(
191+
instance_type="node",
192+
sources=sources,
193+
space=space,
194+
sort=sort,
195+
filter=filter,
196+
limit=limit,
197+
)
198+
if strip:
199+
for node in results:
200+
node.drop_source(COGNITE_FILE_VIEW_ID)
201+
return results

‎cognite/client/_api/files.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -495,7 +495,7 @@ async def upload_content(
495495
Upload file content from a local file path to a file previously created (initiated) with only metadata.
496496
For files created with FilesAPI.create(), use `external_id`.
497497
For files created with data modeling API using CogniteFileApply, use `instance_id`.
498-
Supports upload of large files (>5 GB), using multipart upload.
498+
Supports upload of large files (>5 GiB), using multipart upload.
499499
500500
Args:
501501
path (Path | str): Local file path.
@@ -544,7 +544,7 @@ async def upload(
544544
Note:
545545
If path is a directory, this method will upload all files in that directory. Use `recursive=True` for subdirectories as well.
546546
547-
Supports upload of large files (>5 GB), using multipart upload.
547+
Supports upload of large files (>5 GiB), using multipart upload.
548548
549549
Args:
550550
path (Path | str): Path to the file you wish to upload. If path is a directory, this method will upload all files in that directory.

‎cognite/client/_sync_api/data_modeling/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""
22
===============================================================================
3-
b80632521fdada43a35c314cc92b055b
3+
048014222247324af5e2bdcfee2e1b58
44
This file is auto-generated from the Async API modules, - do not edit manually!
55
===============================================================================
66
"""
@@ -9,8 +9,10 @@
99

1010
from typing import TYPE_CHECKING
1111

12+
from cognite.client import AsyncCogniteClient
1213
from cognite.client._sync_api.data_modeling.containers import SyncContainersAPI
1314
from cognite.client._sync_api.data_modeling.data_models import SyncDataModelsAPI
15+
from cognite.client._sync_api.data_modeling.files import SyncDataModelingFilesAPI
1416
from cognite.client._sync_api.data_modeling.graphql import SyncDataModelingGraphQLAPI
1517
from cognite.client._sync_api.data_modeling.instances import SyncInstancesAPI
1618
from cognite.client._sync_api.data_modeling.records import SyncRecordsAPI
@@ -31,6 +33,7 @@ def __init__(self, async_client: AsyncCogniteClient) -> None:
3133
self.__async_client = async_client
3234
self.containers = SyncContainersAPI(async_client)
3335
self.data_models = SyncDataModelsAPI(async_client)
36+
self.files = SyncDataModelingFilesAPI(async_client)
3437
self.spaces = SyncSpacesAPI(async_client)
3538
self.views = SyncViewsAPI(async_client)
3639
self.instances = SyncInstancesAPI(async_client)

0 commit comments

Comments
 (0)