Python(feat): Remote Files API (#366)

Added Remote Files API to the python client.

Author: andrew-siftstack

python/lib/sift_client/_internal/low_level_wrappers/__init__.py

@@ -5,6 +5,7 @@
 from sift_client._internal.low_level_wrappers.channels import ChannelsLowLevelClient
 from sift_client._internal.low_level_wrappers.ingestion import IngestionLowLevelClient
 from sift_client._internal.low_level_wrappers.ping import PingLowLevelClient
+from sift_client._internal.low_level_wrappers.remote_files import RemoteFilesLowLevelClient
 from sift_client._internal.low_level_wrappers.reports import ReportsLowLevelClient
 from sift_client._internal.low_level_wrappers.rules import RulesLowLevelClient
 from sift_client._internal.low_level_wrappers.runs import RunsLowLevelClient
@@ -18,6 +19,7 @@
     "ChannelsLowLevelClient",
     "IngestionLowLevelClient",
     "PingLowLevelClient",
+    "RemoteFilesLowLevelClient",
     "ReportsLowLevelClient",
     "RulesLowLevelClient",
     "RunsLowLevelClient",

python/lib/sift_client/_internal/low_level_wrappers/remote_files.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Any, cast
+
+import requests
+from sift.remote_files.v1.remote_files_pb2 import (
+    BatchDeleteRemoteFilesRequest,
+    DeleteRemoteFileRequest,
+    GetRemoteFileDownloadUrlRequest,
+    GetRemoteFileRequest,
+    GetRemoteFileResponse,
+    ListRemoteFilesRequest,
+    ListRemoteFilesResponse,
+    UpdateRemoteFileRequest,
+    UpdateRemoteFileResponse,
+)
+from sift.remote_files.v1.remote_files_pb2_grpc import RemoteFileServiceStub
+
+from sift_client._internal.low_level_wrappers.base import (
+    LowLevelClientBase,
+)
+from sift_client.transport import GrpcClient, WithGrpcClient
+
+if TYPE_CHECKING:
+    from sift_client.client import SiftClient
+    from sift_client.sift_types.file_attachment import FileAttachment, FileAttachmentUpdate
+
+
+class RemoteFilesLowLevelClient(LowLevelClientBase, WithGrpcClient):
+    """Low-level client for the RemoteFilesAPI.
+
+    This class provides a thin wrapper around the autogenerated bindings for the RemoteFilesAPI.
+    """
+
+    def __init__(self, grpc_client: GrpcClient):
+        """Initialize the RemoteFilesLowLevelClient.
+
+        Args:
+            grpc_client: The gRPC client to use for making API calls.
+        """
+        super().__init__(grpc_client)
+
+    async def get_remote_file(
+        self, remote_file_id: str, sift_client: SiftClient | None = None
+    ) -> FileAttachment:
+        """Get a remote file by ID.
+
+        Args:
+            remote_file_id: The ID of the remote file to retrieve.
+            sift_client: The SiftClient to attach to the returned RemoteFile.
+
+        Returns:
+            The RemoteFile.
+        """
+        from sift_client.sift_types.file_attachment import FileAttachment
+
+        request = GetRemoteFileRequest(remote_file_id=remote_file_id)
+        response = await self._grpc_client.get_stub(RemoteFileServiceStub).GetRemoteFile(request)
+        grpc_remote_file = cast("GetRemoteFileResponse", response).remote_file
+        return FileAttachment._from_proto(grpc_remote_file, sift_client)
+
+    async def list_all_remote_files(
+        self,
+        query_filter: str | None = None,
+        max_results: int | None = None,
+        page_size: int | None = None,
+        order_by: str | None = None,
+        sift_client: SiftClient | None = None,
+    ) -> list[FileAttachment]:
+        """List all remote files matching the given query.
+
+        Args:
+            query_filter: The CEL query filter.
+
+            max_results: The maximum number of results to return.
+            page_size: The number of results to return per page.
+            order_by: The field to order by. Not supported by the backend, but it is here for API consistency. TODO: Add to backend
+            sift_client: The SiftClient to attach to the returned RemoteFiles.
+
+        Returns:
+            A list of RemoteFiles matching the given query.
+        """
+        return await self._handle_pagination(
+            self.list_remote_files,
+            kwargs={"query_filter": query_filter, "sift_client": sift_client},
+            page_size=page_size,
+            max_results=max_results,
+            order_by=order_by,
+        )
+
+    async def list_remote_files(
+        self,
+        page_size: int | None = None,
+        page_token: str | None = None,
+        query_filter: str | None = None,
+        order_by: str | None = None,
+        sift_client: SiftClient | None = None,
+    ) -> tuple[list[FileAttachment], str]:
+        """List remote files with pagination support.
+
+        Args:
+            page_size: The number of results to return per page.
+            page_token: The page token for pagination.
+            query_filter: The CEL query filter.
+            order_by: The field to order by. Not supported by the backend, but it is here for API consistency. TODO: Add to backend
+            sift_client: The SiftClient to attach to the returned RemoteFiles.
+
+        Returns:
+            A tuple of (list of RemoteFiles, next_page_token).
+        """
+        from sift_client.sift_types.file_attachment import FileAttachment
+
+        if order_by is not None:
+            raise NotImplementedError
+
+        request_kwargs: dict[str, Any] = {}
+        if page_size is not None:
+            request_kwargs["page_size"] = page_size
+        if page_token is not None:
+            request_kwargs["page_token"] = page_token
+        if query_filter is not None:
+            request_kwargs["filter"] = query_filter
+
+        request = ListRemoteFilesRequest(**request_kwargs)
+        response = await self._grpc_client.get_stub(RemoteFileServiceStub).ListRemoteFiles(request)
+        response = cast("ListRemoteFilesResponse", response)
+        return [
+            FileAttachment._from_proto(rf, sift_client) for rf in response.remote_files
+        ], response.next_page_token
+
+    async def update_remote_file(
+        self, update: FileAttachmentUpdate, sift_client: SiftClient | None = None
+    ) -> FileAttachment:
+        """Update a remote file.
+
+        Args:
+            update: The FileAttachmentUpdate containing the fields to update.
+            sift_client: The SiftClient to attach to the returned RemoteFile.
+
+        Returns:
+            The updated RemoteFile.
+        """
+        from sift_client.sift_types.file_attachment import FileAttachment
+
+        grpc_remote_file, update_mask = update.to_proto_with_mask()
+        request = UpdateRemoteFileRequest(remote_file=grpc_remote_file, update_mask=update_mask)
+        response = await self._grpc_client.get_stub(RemoteFileServiceStub).UpdateRemoteFile(request)
+        updated_grpc_remote_file = cast("UpdateRemoteFileResponse", response).remote_file
+        return FileAttachment._from_proto(updated_grpc_remote_file, sift_client)
+
+    async def delete_remote_file(self, remote_file_id: str) -> None:
+        """Delete a remote file.
+
+        Args:
+            remote_file_id: The ID of the remote file to delete.
+        """
+        request = DeleteRemoteFileRequest(remote_file_id=remote_file_id)
+        await self._grpc_client.get_stub(RemoteFileServiceStub).DeleteRemoteFile(request)
+
+    async def batch_delete_remote_files(self, remote_file_ids: list[str]) -> None:
+        """Batch delete remote files.
+
+        Args:
+            remote_file_ids: The IDs of the remote files to delete (up to 1000).
+        """
+        request = BatchDeleteRemoteFilesRequest(remote_file_ids=remote_file_ids)
+        await self._grpc_client.get_stub(RemoteFileServiceStub).BatchDeleteRemoteFiles(request)
+
+    async def get_remote_file_download_url(self, remote_file_id: str) -> str:
+        """Get a download URL for a remote file.
+
+        Args:
+            remote_file_id: The ID of the remote file.
+
+        Returns:
+            The download URL for the remote file.
+        """
+        request = GetRemoteFileDownloadUrlRequest(remote_file_id=remote_file_id)
+        response = await self._grpc_client.get_stub(RemoteFileServiceStub).GetRemoteFileDownloadUrl(
+            request
+        )
+        return response.download_url
+
+    async def download_remote_file(self, file_attachment: FileAttachment) -> bytes:
+        """Download a remote file.
+
+        Args:
+            file_attachment: The FileAttachment to download.
+
+        Returns:
+            The downloaded file.
+        """
+        url = await self.get_remote_file_download_url(file_attachment._id_or_error)
+
+        # Run the synchronous requests.get in a thread pool to avoid blocking
+        def _download():
+            response = requests.get(url)
+            response.raise_for_status()
+            return response.content
+
+        # Use run_in_executor for Python 3.8 compatibility (asyncio.to_thread was added in 3.9)
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, _download)

python/lib/sift_client/_tests/README.md

@@ -0,0 +1,13 @@
+# Sift Python
+
+## Running Integration Tests Locally
+
+1. Create Environmental Variables
+   a. Create or open a .env file in /python
+   b. Add an API key for SIFT_API_KEY
+2. Start Local Sift
+3. Asset Data: NostromoLV426
+   a. Ensure that your local Sift instance contains data for the asset NostromoLV426
+   b. If it doesn't them export data for NostromoLV426 from development
+4. Run tests
+   a. Run tests using /python/scripts/dev {test, test-integration, test-all}

python/lib/sift_client/_tests/conftest.py

@@ -19,13 +19,15 @@ def sift_client() -> SiftClient:
     grpc_url = os.getenv("SIFT_GRPC_URI", "localhost:50051")
     rest_url = os.getenv("SIFT_REST_URI", "localhost:8080")
     api_key = os.getenv("SIFT_API_KEY", "")
+    # If the URL contains localhost, don't use SSL.  Most likely running tests or local development.
+    use_ssl = not ("localhost" in grpc_url or "localhost" in rest_url)
 
     client = SiftClient(
         connection_config=SiftConnectionConfig(
             api_key=api_key,
             grpc_url=grpc_url,
             rest_url=rest_url,
-            use_ssl=True,
+            use_ssl=use_ssl,
         )
     )
 
@@ -45,6 +47,7 @@ def mock_client():
     client.rules = MagicMock()
     client.tags = MagicMock()
     client.test_results = MagicMock()
+    client.file_attachments = MagicMock()
     client.async_ = MagicMock(spec=AsyncAPIs)
     client.async_.ingestion = MagicMock()
     return client