From b3ffe17f448e5b7eaa73484f3f1486951d2debaf Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Fri, 9 May 2025 00:49:21 -0400 Subject: [PATCH 01/12] consolidate clients into one file, update docs, update imports, change core-async-io module to use sync function creation --- cterasdk/asynchronous/core/files/browser.py | 6 +- cterasdk/asynchronous/core/files/io.py | 6 +- cterasdk/clients/asynchronous/__init__.py | 0 cterasdk/clients/asynchronous/clients.py | 156 -------------- cterasdk/clients/asynchronous/errors.py | 6 - cterasdk/clients/base.py | 35 ++++ cterasdk/clients/{synchronous => }/clients.py | 198 ++++++++++++++---- cterasdk/clients/synchronous/__init__.py | 0 cterasdk/clients/synchronous/errors.py | 6 - cterasdk/direct/client.py | 2 +- cterasdk/direct/lib.py | 6 +- cterasdk/objects/asynchronous/core.py | 2 +- cterasdk/objects/services.py | 2 +- cterasdk/objects/synchronous/core.py | 2 +- cterasdk/objects/synchronous/drive.py | 2 +- cterasdk/objects/synchronous/edge.py | 2 +- docs/source/UserGuides/Edge/Quickstart.rst | 12 +- docs/source/UserGuides/Portal/Quickstart.rst | 12 +- .../cterasdk.clients.asynchronous.clients.rst | 7 - .../cterasdk.clients.asynchronous.errors.rst | 7 - .../api/cterasdk.clients.asynchronous.rst | 15 -- docs/source/api/cterasdk.clients.rst | 2 - .../cterasdk.clients.synchronous.clients.rst | 7 - .../cterasdk.clients.synchronous.errors.rst | 7 - .../api/cterasdk.clients.synchronous.rst | 15 -- tests/ut/core/admin/test_login.py | 2 +- tests/ut/core/admin/test_remote.py | 2 +- tests/ut/edge/test_login.py | 2 +- 28 files changed, 222 insertions(+), 299 deletions(-) delete mode 100644 cterasdk/clients/asynchronous/__init__.py delete mode 100644 cterasdk/clients/asynchronous/clients.py delete mode 100644 cterasdk/clients/asynchronous/errors.py rename cterasdk/clients/{synchronous => }/clients.py (56%) delete mode 100644 cterasdk/clients/synchronous/__init__.py delete mode 100644 cterasdk/clients/synchronous/errors.py delete mode 100644 docs/source/api/cterasdk.clients.asynchronous.clients.rst delete mode 100644 docs/source/api/cterasdk.clients.asynchronous.errors.rst delete mode 100644 docs/source/api/cterasdk.clients.asynchronous.rst delete mode 100644 docs/source/api/cterasdk.clients.synchronous.clients.rst delete mode 100644 docs/source/api/cterasdk.clients.synchronous.errors.rst delete mode 100644 docs/source/api/cterasdk.clients.synchronous.rst diff --git a/cterasdk/asynchronous/core/files/browser.py b/cterasdk/asynchronous/core/files/browser.py index 273518f3..405d4d40 100644 --- a/cterasdk/asynchronous/core/files/browser.py +++ b/cterasdk/asynchronous/core/files/browser.py @@ -16,7 +16,7 @@ async def handle(self, path): :param str path: Path to a file """ - handle_function = await io.handle(self.normalize(path)) + handle_function = io.handle(self.normalize(path)) return await handle_function(self._core) async def handle_many(self, directory, *objects): @@ -26,7 +26,7 @@ async def handle_many(self, directory, *objects): :param str directory: Path to a folder :param args objects: List of files and folders """ - handle_many_function = await io.handle_many(self.normalize(directory), *objects) + handle_many_function = io.handle_many(self.normalize(directory), *objects) return await handle_many_function(self._core) async def download(self, path, destination=None): @@ -136,7 +136,7 @@ async def upload(self, name, size, destination, handle): :param str destination: Path to remote directory. :param object handle: Handle. """ - upload_function = await io.upload(name, size, self.normalize(destination), handle) + upload_function = io.upload(name, size, self.normalize(destination), handle) return await upload_function(self._core) async def upload_file(self, path, destination): diff --git a/cterasdk/asynchronous/core/files/io.py b/cterasdk/asynchronous/core/files/io.py index 25bae76e..358605e5 100644 --- a/cterasdk/asynchronous/core/files/io.py +++ b/cterasdk/asynchronous/core/files/io.py @@ -87,7 +87,7 @@ async def retrieve_remote_dir(core, directory): return str(resource.cloudFolderInfo.uid) -async def handle(path): +def handle(path): """ Create function to retrieve file handle. @@ -106,7 +106,7 @@ async def wrapper(core): return wrapper -async def handle_many(directory, *objects): +def handle_many(directory, *objects): """ Create function to retrieve zip archive @@ -128,7 +128,7 @@ async def wrapper(core): return wrapper -async def upload(name, size, destination, fd): +def upload(name, size, destination, fd): """ Create upload function diff --git a/cterasdk/clients/asynchronous/__init__.py b/cterasdk/clients/asynchronous/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cterasdk/clients/asynchronous/clients.py b/cterasdk/clients/asynchronous/clients.py deleted file mode 100644 index d726acd8..00000000 --- a/cterasdk/clients/asynchronous/clients.py +++ /dev/null @@ -1,156 +0,0 @@ -from . import errors -from ..base import BaseClient, BaseResponse -from ..common import Serializers, Deserializers -from .. import async_requests, decorators -from ...common import Object - - -class AsyncClient(BaseClient): - """Asynchronous Client""" - - @decorators.authenticated - async def get(self, path, *, on_response=None, **kwargs): - request = async_requests.GetRequest(self._builder(path), **kwargs) - return await self.async_request(request, on_response=on_response) - - @decorators.authenticated - async def put(self, path, data, *, data_serializer=None, on_response=None, **kwargs): - request = async_requests.PutRequest(self._builder(path), data=data_serializer(data), **kwargs) - return await self.async_request(request, on_response=on_response) - - @decorators.authenticated - async def post(self, path, data, *, data_serializer=None, on_response=None, **kwargs): - request = async_requests.PostRequest(self._builder(path), data=data_serializer(data), **kwargs) - return await self.async_request(request, on_response=on_response) - - @decorators.authenticated - async def form_data(self, path, data, *, on_response=None, **kwargs): - request = async_requests.PostRequest(self._builder(path), data=Serializers.FormData(data), **kwargs) - return await self.async_request(request, on_response=on_response) - - @decorators.authenticated - async def delete(self, path, *, on_response=None, **kwargs): - request = async_requests.DeleteRequest(self._builder(path), **kwargs) - return await self.async_request(request, on_response=on_response) - - async def _request(self, request, *, on_response=None): - on_response = on_response if on_response else AsyncResponse.new() - response = await self._session.await_promise(self.join_headers(request), on_response=on_response) - return await errors.accept(response) - - -class AsyncFolders(AsyncClient): - - async def download_zip(self, path, data, **kwargs): - return await super().form_data(path, data, **kwargs) - - -class AsyncUpload(AsyncClient): - - async def upload(self, path, data, **kwargs): - return await super().form_data(path, data, **kwargs) - - -class AsyncWebDAV(AsyncClient): - """WebDAV""" - - -class AsyncJSON(AsyncClient): - - def __init__(self, builder=None, session=None, settings=None, authenticator=None): - super().__init__(builder, session, settings, authenticator) - self.headers.persist_headers({'Content-Type': 'application/json'}) - - async def get(self, path, **kwargs): - response = await super().get(path, **kwargs) - return await response.json() - - async def put(self, path, data, **kwargs): - response = await super().put(path, data, data_serializer=Serializers.JSON, **kwargs) - return await response.json() - - async def post(self, path, data, **kwargs): - response = await super().post(path, data, data_serializer=Serializers.JSON, **kwargs) - return await response.json() - - async def delete(self, path, **kwargs): - response = await super().delete(path, **kwargs) - return await response.json() - - -class AsyncXML(AsyncClient): - - async def get(self, path, **kwargs): - response = await super().get(path, **kwargs) - return await response.xml() - - async def put(self, path, data, **kwargs): - response = await super().put(path, data, data_serializer=Serializers.XML, **kwargs) - return await response.xml() - - async def post(self, path, data, **kwargs): - response = await super().post(path, data, data_serializer=Serializers.XML, **kwargs) - return await response.xml() - - async def delete(self, path, **kwargs): - response = await super().delete(path, **kwargs) - return await response.xml() - - -class AsyncExtended(AsyncXML): - """CTERA Schema""" - - async def get_multi(self, path, paths): - return await self.database(path, 'get-multi', paths) - - async def execute(self, path, name, param=None): # schema method - return await self._execute(path, 'user-defined', name, param) - - async def database(self, path, name, param=None): # schema method - return await self._execute(path, 'db', name, param) - - async def _execute(self, path, _type, name, param): - data = Object() - data.type = _type - data.name = name - data.param = param - return await super().post(path, data) - - -class AsyncAPI(AsyncExtended): - """CTERA Management API""" - - async def web_session(self): - response = await AsyncClient.get(self, '/currentSession') - self.headers.persist_response_header(response, 'X-CTERA-TOKEN') - return await response.xml() - - async def defaults(self, classname): - return self.get(f'/defaults/{classname}') - - -class AsyncResponse(BaseResponse): - """Asynchronous Response Object""" - - async def async_iter_content(self, chunk_size=None): - async for chunk in self._response.content.iter_chunked(chunk_size if chunk_size else 5120): - yield chunk - - async def text(self): - return await self._response.text() - - async def json(self): - return Deserializers.JSON(await self._response.read()) - - async def xml(self): - return Deserializers.XML(await self._response.read()) - - @async_requests.decorate_stream_error - async def read(self, n=-1): - return await self._response.content.read(n) - - @staticmethod - def new(): - async def new_response(response): - return AsyncResponse(response) - return new_response diff --git a/cterasdk/clients/asynchronous/errors.py b/cterasdk/clients/asynchronous/errors.py deleted file mode 100644 index 9474f9d9..00000000 --- a/cterasdk/clients/asynchronous/errors.py +++ /dev/null @@ -1,6 +0,0 @@ -from .. import errors - - -async def accept(response): - error_message = await response.text() if response.status > 399 else None - return errors.accept(response, error_message) diff --git a/cterasdk/clients/base.py b/cterasdk/clients/base.py index 013ae124..638b2d2e 100644 --- a/cterasdk/clients/base.py +++ b/cterasdk/clients/base.py @@ -1,4 +1,5 @@ import logging +import threading from . import async_requests from ..common import utils @@ -145,3 +146,37 @@ def headers(self): def raise_for_status(self): return self._response.raise_for_status() + + +def run_threadsafe(loop, target, *args, **kwargs): + event = threading.Event() + + t = Task(loop, event, target, *args, **kwargs) + t.start() + + event.wait() + + if t.exception: + raise t.exception + return t.response + + +class Task(threading.Thread): + + def __init__(self, loop, event, target, *args, **kwargs): + super().__init__(name='Thread-safe Executor') + self.loop = loop + self.event = event + self.target = target + self.args = args + self.kwargs = kwargs + self.exception = None + self.response = None + + def run(self): + try: + self.response = self.loop.run_until_complete(self.target(*self.args, **self.kwargs)) + except Exception as e: # pylint: disable=broad-exception-caught + self.exception = e + finally: + self.event.set() diff --git a/cterasdk/clients/synchronous/clients.py b/cterasdk/clients/clients.py similarity index 56% rename from cterasdk/clients/synchronous/clients.py rename to cterasdk/clients/clients.py index e666f2e0..dddd6622 100644 --- a/cterasdk/clients/synchronous/clients.py +++ b/cterasdk/clients/clients.py @@ -1,12 +1,161 @@ - import asyncio -import threading from . import errors -from ..base import BaseClient -from ..common import Serializers -from ..asynchronous.clients import AsyncResponse -from ...common import Object +from .base import BaseClient, BaseResponse, run_threadsafe +from ..common import Serializers, Deserializers from .. import async_requests, decorators +from ..common import Object + + +class AsyncClient(BaseClient): + """Asynchronous Client""" + + @decorators.authenticated + async def get(self, path, *, on_response=None, **kwargs): + request = async_requests.GetRequest(self._builder(path), **kwargs) + return await self.async_request(request, on_response=on_response) + + @decorators.authenticated + async def put(self, path, data, *, data_serializer=None, on_response=None, **kwargs): + request = async_requests.PutRequest(self._builder(path), data=data_serializer(data), **kwargs) + return await self.async_request(request, on_response=on_response) + + @decorators.authenticated + async def post(self, path, data, *, data_serializer=None, on_response=None, **kwargs): + request = async_requests.PostRequest(self._builder(path), data=data_serializer(data), **kwargs) + return await self.async_request(request, on_response=on_response) + + @decorators.authenticated + async def form_data(self, path, data, *, on_response=None, **kwargs): + request = async_requests.PostRequest(self._builder(path), data=Serializers.FormData(data), **kwargs) + return await self.async_request(request, on_response=on_response) + + @decorators.authenticated + async def delete(self, path, *, on_response=None, **kwargs): + request = async_requests.DeleteRequest(self._builder(path), **kwargs) + return await self.async_request(request, on_response=on_response) + + async def _request(self, request, *, on_response=None): + on_response = on_response if on_response else AsyncResponse.new() + response = await self._session.await_promise(self.join_headers(request), on_response=on_response) + error_message = await response.text() if response.status > 399 else None + return errors.accept(response, error_message) + + +class AsyncFolders(AsyncClient): + + async def download_zip(self, path, data, **kwargs): + return await super().form_data(path, data, **kwargs) + + +class AsyncUpload(AsyncClient): + + async def upload(self, path, data, **kwargs): + return await super().form_data(path, data, **kwargs) + + +class AsyncWebDAV(AsyncClient): + """WebDAV""" + + +class AsyncJSON(AsyncClient): + + def __init__(self, builder=None, session=None, settings=None, authenticator=None): + super().__init__(builder, session, settings, authenticator) + self.headers.persist_headers({'Content-Type': 'application/json'}) + + async def get(self, path, **kwargs): + response = await super().get(path, **kwargs) + return await response.json() + + async def put(self, path, data, **kwargs): + response = await super().put(path, data, data_serializer=Serializers.JSON, **kwargs) + return await response.json() + + async def post(self, path, data, **kwargs): + response = await super().post(path, data, data_serializer=Serializers.JSON, **kwargs) + return await response.json() + + async def delete(self, path, **kwargs): + response = await super().delete(path, **kwargs) + return await response.json() + + +class AsyncXML(AsyncClient): + + async def get(self, path, **kwargs): + response = await super().get(path, **kwargs) + return await response.xml() + + async def put(self, path, data, **kwargs): + response = await super().put(path, data, data_serializer=Serializers.XML, **kwargs) + return await response.xml() + + async def post(self, path, data, **kwargs): + response = await super().post(path, data, data_serializer=Serializers.XML, **kwargs) + return await response.xml() + + async def delete(self, path, **kwargs): + response = await super().delete(path, **kwargs) + return await response.xml() + + +class AsyncExtended(AsyncXML): + """CTERA Schema""" + + async def get_multi(self, path, paths): + return await self.database(path, 'get-multi', paths) + + async def execute(self, path, name, param=None): # schema method + return await self._execute(path, 'user-defined', name, param) + + async def database(self, path, name, param=None): # schema method + return await self._execute(path, 'db', name, param) + + async def _execute(self, path, _type, name, param): + data = Object() + data.type = _type + data.name = name + data.param = param + return await super().post(path, data) + + +class AsyncAPI(AsyncExtended): + """CTERA Management API""" + + async def web_session(self): + response = await AsyncClient.get(self, '/currentSession') + self.headers.persist_response_header(response, 'X-CTERA-TOKEN') + return await response.xml() + + async def defaults(self, classname): + return self.get(f'/defaults/{classname}') + + +class AsyncResponse(BaseResponse): + """Asynchronous Response Object""" + + async def async_iter_content(self, chunk_size=None): + async for chunk in self._response.content.iter_chunked(chunk_size if chunk_size else 5120): + yield chunk + + async def text(self): + return await self._response.text() + + async def json(self): + return Deserializers.JSON(await self._response.read()) + + async def xml(self): + return Deserializers.XML(await self._response.read()) + + @async_requests.decorate_stream_error + async def read(self, n=-1): + return await self._response.content.read(n) + + @staticmethod + def new(): + async def new_response(response): + return AsyncResponse(response) + return new_response class Client(BaseClient): @@ -50,7 +199,8 @@ def delete(self, path, *, on_response=None, **kwargs): def _request(self, request, *, on_response=None): on_response = on_response if on_response else SyncResponse.new() response = execute(self._session.await_promise, self.join_headers(request), on_response=on_response) - return errors.accept(response) + error_message = response.text() if response.status > 399 else None + return errors.accept(response, error_message) def close(self): # pylint: disable=invalid-overridden-method return execute(super().close) @@ -230,37 +380,3 @@ def new(): async def new_response(response): return SyncResponse(response) return new_response - - -def run_threadsafe(loop, target, *args, **kwargs): - event = threading.Event() - - t = Task(loop, event, target, *args, **kwargs) - t.start() - - event.wait() - - if t.exception: - raise t.exception - return t.response - - -class Task(threading.Thread): - - def __init__(self, loop, event, target, *args, **kwargs): - super().__init__(name='Thread-safe Executor') - self.loop = loop - self.event = event - self.target = target - self.args = args - self.kwargs = kwargs - self.exception = None - self.response = None - - def run(self): - try: - self.response = self.loop.run_until_complete(self.target(*self.args, **self.kwargs)) - except Exception as e: # pylint: disable=broad-exception-caught - self.exception = e - finally: - self.event.set() diff --git a/cterasdk/clients/synchronous/__init__.py b/cterasdk/clients/synchronous/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cterasdk/clients/synchronous/errors.py b/cterasdk/clients/synchronous/errors.py deleted file mode 100644 index fec82b8e..00000000 --- a/cterasdk/clients/synchronous/errors.py +++ /dev/null @@ -1,6 +0,0 @@ -from .. import errors - - -def accept(response): - error_message = response.text() if response.status > 399 else None - return errors.accept(response, error_message) diff --git a/cterasdk/direct/client.py b/cterasdk/direct/client.py index cc02e613..ff3f6aad 100644 --- a/cterasdk/direct/client.py +++ b/cterasdk/direct/client.py @@ -9,7 +9,7 @@ from ..objects.endpoints import DefaultBuilder, EndpointBuilder from ..clients.settings import ClientSettings, ClientTimeout, TCPConnector -from ..clients.asynchronous.clients import AsyncClient, AsyncJSON +from ..clients.clients import AsyncClient, AsyncJSON def client_settings(parameters): diff --git a/cterasdk/direct/lib.py b/cterasdk/direct/lib.py index 6ee00673..7adeeb15 100644 --- a/cterasdk/direct/lib.py +++ b/cterasdk/direct/lib.py @@ -108,7 +108,7 @@ async def process_chunk(client, file_id, chunk, encryption_key, semaphore): """ Process a Chunk. - :param cterasdk.clients.asynchronous.clients.AsyncClient client: Asynchronous HTTP Client. + :param cterasdk.clients.clients.AsyncClient client: Asynchronous HTTP Client. :param int file_id: File ID. :param cterasdk.direct.types.Chunk chunk: Chunk. :param str encryption_key: Encryption key. @@ -136,7 +136,7 @@ async def process_chunks(client, file_id, chunks, encryption_key, semaphore=None """ Process Chunks Asynchronously. - :param cterasdk.clients.asynchronous.clients.AsyncClient client: Asynchronous HTTP Client. + :param cterasdk.clients.clients.AsyncClient client: Asynchronous HTTP Client. :param int file_id: File ID. :param list[cterasdk.direct.types.Chunk] chunks: Chunk. :param str encryption_key: Encryption key. @@ -194,7 +194,7 @@ async def get_chunks(api, credentials, file_id): """ Get Chunks. - :param cterasdk.clients.asynchronous.clients.AsyncJSON api: Asynchronous JSON Client. + :param cterasdk.clients.clients.AsyncJSON api: Asynchronous JSON Client. :param int file_id: File ID. :returns: Wrapped key and file chunks. :rtype: cterasdk.direct.types.DirectIOResponse diff --git a/cterasdk/objects/asynchronous/core.py b/cterasdk/objects/asynchronous/core.py index 07497aef..df3be025 100644 --- a/cterasdk/objects/asynchronous/core.py +++ b/cterasdk/objects/asynchronous/core.py @@ -1,7 +1,7 @@ import cterasdk.settings from ..services import CTERA, client_settings from ..endpoints import EndpointBuilder -from ...clients.asynchronous import clients +from ...clients import clients from .. import authenticators diff --git a/cterasdk/objects/services.py b/cterasdk/objects/services.py index 6832316e..cb2dfd84 100644 --- a/cterasdk/objects/services.py +++ b/cterasdk/objects/services.py @@ -4,7 +4,7 @@ from . import endpoints, uri from .utils import URI from ..clients.settings import ClientSettings, ClientTimeout, TCPConnector, CookieJar -from ..clients.synchronous import clients +from ..clients.clients import clients from ..common import Object from ..convert import tojsonstr diff --git a/cterasdk/objects/synchronous/core.py b/cterasdk/objects/synchronous/core.py index 34a0ecb7..1cd11a09 100644 --- a/cterasdk/objects/synchronous/core.py +++ b/cterasdk/objects/synchronous/core.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from ...clients.synchronous import clients +from ...clients.clients import clients from ..services import Management from ..endpoints import EndpointBuilder diff --git a/cterasdk/objects/synchronous/drive.py b/cterasdk/objects/synchronous/drive.py index 56793699..ccadaa8f 100644 --- a/cterasdk/objects/synchronous/drive.py +++ b/cterasdk/objects/synchronous/drive.py @@ -1,4 +1,4 @@ -from ...clients.synchronous import clients +from ...clients.clients import clients from ..services import Management from ..endpoints import EndpointBuilder diff --git a/cterasdk/objects/synchronous/edge.py b/cterasdk/objects/synchronous/edge.py index cc29ae13..036fe04f 100644 --- a/cterasdk/objects/synchronous/edge.py +++ b/cterasdk/objects/synchronous/edge.py @@ -1,4 +1,4 @@ -from ...clients.synchronous import clients +from ...clients.clients import clients from ..services import Management from ..endpoints import EndpointBuilder diff --git a/docs/source/UserGuides/Edge/Quickstart.rst b/docs/source/UserGuides/Edge/Quickstart.rst index 3e3b26cf..bd70d0d5 100644 --- a/docs/source/UserGuides/Edge/Quickstart.rst +++ b/docs/source/UserGuides/Edge/Quickstart.rst @@ -113,7 +113,7 @@ The ``Edge`` object features an ``api`` property used for accessing *Core Method .. warning:: For optimal integration, it's advised to utilize the modules provided in this SDK instead of the ``api`` property. In cases where a specific command or module is absent, `please submit a feature request `_. -.. automethod:: cterasdk.clients.synchronous.clients.API.get +.. automethod:: cterasdk.clients.clients.API.get :noindex: .. code-block:: python @@ -121,10 +121,10 @@ The ``Edge`` object features an ``api`` property used for accessing *Core Method hostname = edge.api.get('/config/device/hostname') # Not recommended hostname = edge.config.get_hostname() # Recommended: using the config module and the get_hostname() command -.. automethod:: cterasdk.clients.synchronous.clients.API.get_multi +.. automethod:: cterasdk.clients.clients.API.get_multi :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.put +.. automethod:: cterasdk.clients.clients.API.put :noindex: .. code-block:: python @@ -132,7 +132,7 @@ The ``Edge`` object features an ``api`` property used for accessing *Core Method hostname = edge.api.put('/config/device/hostname', 'edge-filesystem') # Not recommended hostname = edge.config.set_hostname('edge-filesystem') # Recommended: using the config module and the set_hostname() command -.. automethod:: cterasdk.clients.synchronous.clients.API.add +.. automethod:: cterasdk.clients.clients.API.add :noindex: .. code-block:: python @@ -151,7 +151,7 @@ The ``Edge`` object features an ``api`` property used for accessing *Core Method """Recommended way of adding a local user""" edge.users.add('alice', 'secret-password', 'Alice Wonderland', 'alice.wonderland@acme.com', 501) -.. automethod:: cterasdk.clients.synchronous.clients.API.execute +.. automethod:: cterasdk.clients.clients.API.execute :noindex: .. code-block:: python @@ -159,7 +159,7 @@ The ``Edge`` object features an ``api`` property used for accessing *Core Method edge.api.execute('/config/cloudsync', 'forceExecuteEvictor') # Not recommended: Start the cache eviction process (force) edge.cache.force_eviction() # Recommended -.. automethod:: cterasdk.clients.synchronous.clients.API.delete +.. automethod:: cterasdk.clients.clients.API.delete :noindex: .. code-block:: python diff --git a/docs/source/UserGuides/Portal/Quickstart.rst b/docs/source/UserGuides/Portal/Quickstart.rst index 24897d7c..eed8d395 100644 --- a/docs/source/UserGuides/Portal/Quickstart.rst +++ b/docs/source/UserGuides/Portal/Quickstart.rst @@ -138,22 +138,22 @@ The ``GlobalAdmin`` and ``ServicesPortal`` objects feature an ``api`` property u .. warning:: For optimal integration, it's advised to utilize the modules provided in this SDK instead of the ``api`` property. In cases where a specific command or module is absent, `please submit a feature request `_. -.. automethod:: cterasdk.clients.synchronous.clients.API.get +.. automethod:: cterasdk.clients.clients.API.get :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.get_multi +.. automethod:: cterasdk.clients.clients.API.get_multi :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.put +.. automethod:: cterasdk.clients.clients.API.put :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.add +.. automethod:: cterasdk.clients.clients.API.add :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.execute +.. automethod:: cterasdk.clients.clients.API.execute :noindex: -.. automethod:: cterasdk.clients.synchronous.clients.API.delete +.. automethod:: cterasdk.clients.clients.API.delete :noindex: Data Types and Enumerators diff --git a/docs/source/api/cterasdk.clients.asynchronous.clients.rst b/docs/source/api/cterasdk.clients.asynchronous.clients.rst deleted file mode 100644 index c675d380..00000000 --- a/docs/source/api/cterasdk.clients.asynchronous.clients.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.clients.asynchronous.clients module -============================================ - -.. automodule:: cterasdk.clients.asynchronous.clients - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.clients.asynchronous.errors.rst b/docs/source/api/cterasdk.clients.asynchronous.errors.rst deleted file mode 100644 index acd850ec..00000000 --- a/docs/source/api/cterasdk.clients.asynchronous.errors.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.clients.asynchronous.errors module -=========================================== - -.. automodule:: cterasdk.clients.asynchronous.errors - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.clients.asynchronous.rst b/docs/source/api/cterasdk.clients.asynchronous.rst deleted file mode 100644 index 3e122374..00000000 --- a/docs/source/api/cterasdk.clients.asynchronous.rst +++ /dev/null @@ -1,15 +0,0 @@ -cterasdk.clients.asynchronous package -===================================== - -.. automodule:: cterasdk.clients.asynchronous - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -.. toctree:: - - cterasdk.clients.asynchronous.clients - cterasdk.clients.asynchronous.errors diff --git a/docs/source/api/cterasdk.clients.rst b/docs/source/api/cterasdk.clients.rst index 19aae08a..2fa62099 100644 --- a/docs/source/api/cterasdk.clients.rst +++ b/docs/source/api/cterasdk.clients.rst @@ -11,12 +11,10 @@ Submodules .. toctree:: - cterasdk.clients.asynchronous cterasdk.clients.async_requests cterasdk.clients.base cterasdk.clients.common cterasdk.clients.decorators cterasdk.clients.errors - cterasdk.clients.synchronous cterasdk.clients.tracers diff --git a/docs/source/api/cterasdk.clients.synchronous.clients.rst b/docs/source/api/cterasdk.clients.synchronous.clients.rst deleted file mode 100644 index bc96bf63..00000000 --- a/docs/source/api/cterasdk.clients.synchronous.clients.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.clients.synchronous.clients module -=========================================== - -.. automodule:: cterasdk.clients.synchronous.clients - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.clients.synchronous.errors.rst b/docs/source/api/cterasdk.clients.synchronous.errors.rst deleted file mode 100644 index 26968f3c..00000000 --- a/docs/source/api/cterasdk.clients.synchronous.errors.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.clients.synchronous.errors module -========================================== - -.. automodule:: cterasdk.clients.synchronous.errors - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.clients.synchronous.rst b/docs/source/api/cterasdk.clients.synchronous.rst deleted file mode 100644 index 8d971bd6..00000000 --- a/docs/source/api/cterasdk.clients.synchronous.rst +++ /dev/null @@ -1,15 +0,0 @@ -cterasdk.clients.synchronous package -==================================== - -.. automodule:: cterasdk.clients.synchronous - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -.. toctree:: - - cterasdk.clients.synchronous.clients - cterasdk.clients.synchronous.errors diff --git a/tests/ut/core/admin/test_login.py b/tests/ut/core/admin/test_login.py index 3df9c94e..8ab208c6 100644 --- a/tests/ut/core/admin/test_login.py +++ b/tests/ut/core/admin/test_login.py @@ -19,7 +19,7 @@ def setUp(self): self._username = TestCoreLogin._username self._password = TestCoreLogin._password self._role = TestCoreLogin._role - self._mock_close = self.patch_call("cterasdk.clients.synchronous.clients.Client.close") + self._mock_close = self.patch_call("cterasdk.clients.clients.Client.close") @staticmethod def _obtain_session_info(path): diff --git a/tests/ut/core/admin/test_remote.py b/tests/ut/core/admin/test_remote.py index 69f00acb..12e05d55 100644 --- a/tests/ut/core/admin/test_remote.py +++ b/tests/ut/core/admin/test_remote.py @@ -18,7 +18,7 @@ def setUp(self): self._user_name = 'user' self._user_role = 'EndUser' self._sso_ticket = 'sso ticket' - self._mock_close = self.patch_call("cterasdk.clients.synchronous.clients.Client.close") + self._mock_close = self.patch_call("cterasdk.clients.clients.Client.close") def test_instantiation_of_remote_devices(self): filer_types = ['CloudPlug', 'C200', 'C400', 'C800', 'C800P', 'vGateway'] diff --git a/tests/ut/edge/test_login.py b/tests/ut/edge/test_login.py index d6ea13ba..ecfb067d 100644 --- a/tests/ut/edge/test_login.py +++ b/tests/ut/edge/test_login.py @@ -13,7 +13,7 @@ def setUp(self): self._username = 'admin' self._password = 'password' self._version = '7.5.182.16' - self._mock_close = self.patch_call("cterasdk.clients.synchronous.clients.Client.close") + self._mock_close = self.patch_call("cterasdk.clients.clients.Client.close") def test_login_success(self): self._init_filer() From b02f55361940394a3e46c611608b9dc56936c042 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Fri, 9 May 2025 00:57:41 -0400 Subject: [PATCH 02/12] resolve module import errors and pass ut --- cterasdk/clients/clients.py | 4 ++-- cterasdk/lib/storage/asynfs.py | 1 - cterasdk/objects/services.py | 2 +- cterasdk/objects/synchronous/core.py | 2 +- cterasdk/objects/synchronous/drive.py | 2 +- cterasdk/objects/synchronous/edge.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cterasdk/clients/clients.py b/cterasdk/clients/clients.py index dddd6622..a640fbef 100644 --- a/cterasdk/clients/clients.py +++ b/cterasdk/clients/clients.py @@ -1,8 +1,8 @@ import asyncio from . import errors from .base import BaseClient, BaseResponse, run_threadsafe -from ..common import Serializers, Deserializers -from .. import async_requests, decorators +from .common import Serializers, Deserializers +from . import async_requests, decorators from ..common import Object diff --git a/cterasdk/lib/storage/asynfs.py b/cterasdk/lib/storage/asynfs.py index e53f22b3..4b004b65 100644 --- a/cterasdk/lib/storage/asynfs.py +++ b/cterasdk/lib/storage/asynfs.py @@ -35,5 +35,4 @@ async def overwrite(p, handle): else: async for chunk in handle.async_iter_content(chunk_size=8192): await fd.write(chunk) - logger.debug('Wrote: %s', p.as_posix()) return p.as_posix() diff --git a/cterasdk/objects/services.py b/cterasdk/objects/services.py index cb2dfd84..2d160c49 100644 --- a/cterasdk/objects/services.py +++ b/cterasdk/objects/services.py @@ -4,7 +4,7 @@ from . import endpoints, uri from .utils import URI from ..clients.settings import ClientSettings, ClientTimeout, TCPConnector, CookieJar -from ..clients.clients import clients +from ..clients import clients from ..common import Object from ..convert import tojsonstr diff --git a/cterasdk/objects/synchronous/core.py b/cterasdk/objects/synchronous/core.py index 1cd11a09..1fe1b45a 100644 --- a/cterasdk/objects/synchronous/core.py +++ b/cterasdk/objects/synchronous/core.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from ...clients.clients import clients +from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder diff --git a/cterasdk/objects/synchronous/drive.py b/cterasdk/objects/synchronous/drive.py index ccadaa8f..8f8a3306 100644 --- a/cterasdk/objects/synchronous/drive.py +++ b/cterasdk/objects/synchronous/drive.py @@ -1,4 +1,4 @@ -from ...clients.clients import clients +from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder diff --git a/cterasdk/objects/synchronous/edge.py b/cterasdk/objects/synchronous/edge.py index 036fe04f..02711f5b 100644 --- a/cterasdk/objects/synchronous/edge.py +++ b/cterasdk/objects/synchronous/edge.py @@ -1,4 +1,4 @@ -from ...clients.clients import clients +from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder From 545540325f11c85118bbea8daacae13b8f947c89 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 18:29:13 -0400 Subject: [PATCH 03/12] remove old filesystem module and update modules, revamp client settings config, add async file extensions for the edge filer --- cterasdk/__init__.py | 3 +- cterasdk/asynchronous/edge/__init__.py | 0 cterasdk/asynchronous/edge/base_command.py | 8 + cterasdk/asynchronous/edge/files/__init__.py | 1 + cterasdk/asynchronous/edge/files/browser.py | 142 ++++++++++ cterasdk/asynchronous/edge/files/io.py | 105 ++++++++ cterasdk/asynchronous/edge/login.py | 35 +++ cterasdk/audit/postman.py | 13 +- cterasdk/clients/async_requests.py | 15 +- cterasdk/clients/base.py | 8 +- cterasdk/clients/clients.py | 28 ++ cterasdk/clients/settings.py | 103 +++++--- cterasdk/clients/tracers/__init__.py | 2 +- cterasdk/clients/tracers/postman.py | 14 +- .../tracers/{logger.py => requests.py} | 21 +- cterasdk/clients/tracers/session.py | 8 +- cterasdk/common/object.py | 3 + cterasdk/core/files/browser.py | 2 +- cterasdk/core/types.py | 4 +- cterasdk/direct/client.py | 15 +- cterasdk/edge/services.py | 4 +- cterasdk/edge/ssh.py | 5 +- cterasdk/edge/support.py | 6 +- cterasdk/lib/__init__.py | 1 - cterasdk/lib/crypto.py | 30 +-- cterasdk/lib/filesystem.py | 249 ------------------ cterasdk/lib/storage/commonfs.py | 2 +- cterasdk/logging.py | 21 -- cterasdk/objects/__init__.py | 1 + cterasdk/objects/asynchronous/core.py | 43 +-- cterasdk/objects/asynchronous/edge.py | 77 ++++++ cterasdk/objects/services.py | 40 ++- cterasdk/objects/synchronous/core.py | 48 +--- cterasdk/objects/synchronous/drive.py | 13 +- cterasdk/objects/synchronous/edge.py | 56 +--- cterasdk/settings.py | 29 +- cterasdk/settings.yml | 106 ++++---- .../UserGuides/DataServices/DirectIO.rst | 12 +- .../DataServices/NotificationService.rst | 6 +- docs/source/UserGuides/Edge/Configuration.rst | 2 +- docs/source/UserGuides/Edge/Quickstart.rst | 8 +- .../source/UserGuides/Miscellaneous/Index.rst | 4 +- docs/source/UserGuides/Portal/Quickstart.rst | 4 +- docs/source/index.rst | 4 +- samples/gateway_sample.py | 148 ----------- samples/portal_simple_sample.py | 84 ------ samples/remove_devices.py | 45 ---- samples/sample_base.py | 19 -- ...rvices_portal_create_directories_sample.py | 40 --- tests/ut/edge/test_config.py | 2 +- tests/ut/edge/test_files_browser.py | 2 +- tests/ut/edge/test_support.py | 4 +- 52 files changed, 680 insertions(+), 965 deletions(-) create mode 100644 cterasdk/asynchronous/edge/__init__.py create mode 100644 cterasdk/asynchronous/edge/base_command.py create mode 100644 cterasdk/asynchronous/edge/files/__init__.py create mode 100644 cterasdk/asynchronous/edge/files/browser.py create mode 100644 cterasdk/asynchronous/edge/files/io.py create mode 100644 cterasdk/asynchronous/edge/login.py rename cterasdk/clients/tracers/{logger.py => requests.py} (70%) delete mode 100644 cterasdk/lib/filesystem.py delete mode 100644 cterasdk/logging.py create mode 100644 cterasdk/objects/asynchronous/edge.py delete mode 100644 samples/gateway_sample.py delete mode 100644 samples/portal_simple_sample.py delete mode 100644 samples/remove_devices.py delete mode 100644 samples/sample_base.py delete mode 100755 samples/services_portal_create_directories_sample.py diff --git a/cterasdk/__init__.py b/cterasdk/__init__.py index 401f694e..474b5ae8 100644 --- a/cterasdk/__init__.py +++ b/cterasdk/__init__.py @@ -1,6 +1,5 @@ # pylint: disable=wrong-import-position import cterasdk.settings # noqa: E402, F401 -import cterasdk.logging # noqa: E402, F401 from .common import Object, PolicyRule # noqa: E402, F401 from .convert import fromjsonstr, tojsonstr, fromxmlstr, toxmlstr # noqa: E402, F401 @@ -12,5 +11,5 @@ from .core import enum as core_enum # noqa: E402, F401 from .common import types as common_types # noqa: E402, F401 from .common import enum as common_enum # noqa: E402, F401 -from .objects import GlobalAdmin, ServicesPortal, Edge, Drive, AsyncGlobalAdmin, AsyncServicesPortal # noqa: E402, F401 +from .objects import GlobalAdmin, ServicesPortal, Edge, Drive, AsyncGlobalAdmin, AsyncServicesPortal, AsyncEdge # noqa: E402, F401 from . import direct as ctera_direct # noqa: E402, F401 diff --git a/cterasdk/asynchronous/edge/__init__.py b/cterasdk/asynchronous/edge/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cterasdk/asynchronous/edge/base_command.py b/cterasdk/asynchronous/edge/base_command.py new file mode 100644 index 00000000..e6c486c0 --- /dev/null +++ b/cterasdk/asynchronous/edge/base_command.py @@ -0,0 +1,8 @@ +class BaseCommand: + """Base Class for CTERA Edge Filer API Commands""" + + def __init__(self, edge): + self._edge = edge + + def session(self): + return self._edge.session() diff --git a/cterasdk/asynchronous/edge/files/__init__.py b/cterasdk/asynchronous/edge/files/__init__.py new file mode 100644 index 00000000..3b5ce04c --- /dev/null +++ b/cterasdk/asynchronous/edge/files/__init__.py @@ -0,0 +1 @@ +from .browser import FileBrowser # noqa: E402, F401 diff --git a/cterasdk/asynchronous/edge/files/browser.py b/cterasdk/asynchronous/edge/files/browser.py new file mode 100644 index 00000000..929b7137 --- /dev/null +++ b/cterasdk/asynchronous/edge/files/browser.py @@ -0,0 +1,142 @@ +from ..base_command import BaseCommand +from ....cio.edge import EdgePath +from ....lib.storage import asynfs, commonfs +from . import io + + +class FileBrowser(BaseCommand): + """ Edge Filer File Browser APIs """ + + async def listdir(self, path): + """ + List Directory + + :param str path: Path + """ + return await io.listdir(self._edge, path) + + async def handle(self, path): + """ + Get File Handle. + + :param str path: Path to a file + """ + handle_function = io.handle(self.normalize(path)) + return await handle_function(self._edge) + + async def handle_many(self, directory, *objects): + """ + Get a Zip Archive File Handle. + + :param str directory: Path to a folder + :param args objects: List of files and folders + """ + handle_many_function = io.handle_many(self.normalize(directory), *objects) + return await handle_many_function(self._edge) + + async def download(self, path, destination=None): + """ + Download a file + + :param str path: The file path on the Edge Filer + :param str,optional destination: + File destination, if it is a directory, the original filename will be kept, defaults to the default directory + """ + directory, name = commonfs.determine_directory_and_filename(path, destination=destination) + handle = await self.handle(path) + return await asynfs.write(directory, name, handle) + + async def download_many(self, target, objects, destination=None): + """ + Download selected files and/or directories as a ZIP archive. + + .. warning:: + The provided list of objects is not validated. Only existing files and directories + will be included in the resulting ZIP file. + + :param str target: + Path to the cloud folder containing the files and directories to download. + :param list[str] objects: + List of file and/or directory names to include in the download. + :param str destination: + Optional. Path to the destination file or directory. If a directory is provided, + the original filename will be preserved. Defaults to the default download directory. + """ + directory, name = commonfs.determine_directory_and_filename(target, objects, destination=destination, archive=True) + handle = await self.handle_many(target, *objects) + return await asynfs.write(directory, name, handle) + + async def upload(self, name, destination, handle): + """ + Upload from file handle. + + :param str name: File name. + :param str destination: Path to remote directory. + :param object handle: Handle. + """ + upload_function = io.upload(name, self.normalize(destination), handle) + return await upload_function(self._edge) + + async def upload_file(self, path, destination): + """ + Upload a file. + + :param str path: Local path + :param str destination: Remote path + """ + metadata = commonfs.properties(path) + with open(path, 'rb') as handle: + response = await self.upload(metadata['name'], destination, handle) + return response + + async def mkdir(self, path): + """ + Create a new directory + + :param str path: Directory path + """ + return await io.mkdir(self._edge, self.normalize(path)) + + async def makedirs(self, path): + """ + Create a directory recursively + + :param str path: Directory path + """ + return await io.makedirs(self._edge, self.normalize(path)) + + async def copy(self, path, destination=None, overwrite=False): + """ + Copy a file or a folder + + :param str path: Source file or folder path + :param str destination: Destination folder path + :param bool,optional overwrite: Overwrite on conflict, defaults to False + """ + if destination is None: + raise ValueError('Copy destination was not specified.') + return await io.copy(self._edge, self.normalize(path), self.normalize(destination), overwrite) + + async def move(self, path, destination=None, overwrite=False): + """ + Move a file or a folder + + :param str path: Source file or folder path + :param str destination: Destination folder path + :param bool,optional overwrite: Overwrite on conflict, defaults to False + """ + if destination is None: + raise ValueError('Move destination was not specified.') + return await io.move(self._edge, self.normalize(path), self.normalize(destination), overwrite) + + async def delete(self, path): + """ + Delete a file + + :param str path: File path + """ + return await io.remove(self._edge, self.normalize(path)) + + @staticmethod + def normalize(path): + return EdgePath('/', path) diff --git a/cterasdk/asynchronous/edge/files/io.py b/cterasdk/asynchronous/edge/files/io.py new file mode 100644 index 00000000..92518f46 --- /dev/null +++ b/cterasdk/asynchronous/edge/files/io.py @@ -0,0 +1,105 @@ +import logging +from ....cio.common import encode_request_parameter +from ....cio import edge as fs +from ....cio import exceptions + + +logger = logging.getLogger('cterasdk.edge') + + +async def listdir(edge, path): + with fs.listdir(path) as param: + return await edge.api.execute('/status/fileManager', 'listPhysicalFolders', param) + + +async def mkdir(edge, path): + with fs.makedir(path) as param: + await edge.io.mkdir(param) + return path.absolute + + +async def makedirs(edge, path): + directories = path.parts + for i in range(1, len(directories) + 1): + path = fs.EdgePath(path.scope, '/'.join(directories[:i])) + try: + await mkdir(edge, path) + except exceptions.RestrictedPathError: + logger.warning('Creating a folder in the specified location is forbidden: %s', path.reference.as_posix()) + + +async def copy(edge, path, destination=None, overwrite=False): + with fs.copy(path, destination) as (src, dst): + await edge.io.copy(src, dst, overwrite=overwrite) + + +async def move(edge, path, destination=None, overwrite=False): + with fs.move(path, destination) as (src, dst): + await edge.io.move(src, dst, overwrite=overwrite) + + +async def remove(edge, *paths): + for path in fs.delete_generator(*paths): + await edge.io.delete(path.absolute) + + +def handle(path): + """ + Create function to retrieve file handle. + + :param cterasdk.cio.edge.EdgePath path: Path to file. + :returns: Callable function to retrieve file handle. + :rtype: callable + """ + async def wrapper(edge): + """ + Get file handle. + + :param cterasdk.objects.synchronous.edge.Edge edge: Edge Filer object. + """ + with fs.handle(path) as param: + return await edge.io.download(param) + return wrapper + + +def handle_many(directory, *objects): + """ + Create function to retrieve zip archive + + :param cterasdk.cio.edge.EdgePath directory: Path to directory. + :param args objects: List of files and folders. + :returns: Callable function to retrieve file handle. + :rtype: callable + """ + async def wrapper(edge): + """ + Upload file from metadata and file handle. + + :param cterasdk.objects.synchronous.edge.Edge edge: Edge Filer object. + :param str name: File name. + :param object handle: File handle. + """ + with fs.handle_many(directory, objects) as param: + return await edge.io.download_zip('/admingui/api/status/fileManager/zip', encode_request_parameter(param)) + return wrapper + + +def upload(name, destination, fd): + """ + Create upload function + + :param str name: File name. + :param cterasdk.cio.edge.EdgePath destination: Path to directory. + :param object fd: File handle. + :returns: Callable function to start the upload. + :rtype: callable + """ + async def wrapper(edge): + """ + Upload file from metadata and file handle. + + :param cterasdk.objects.synchronous.edge.Edge edge: Edge Filer object. + """ + with fs.upload(name, destination, fd) as param: + return await edge.io.upload('/actions/upload', param) + return wrapper diff --git a/cterasdk/asynchronous/edge/login.py b/cterasdk/asynchronous/edge/login.py new file mode 100644 index 00000000..e3ceed5b --- /dev/null +++ b/cterasdk/asynchronous/edge/login.py @@ -0,0 +1,35 @@ +import logging + +from .base_command import BaseCommand +from ...exceptions import CTERAException + + +logger = logging.getLogger('cterasdk.edge') + + +class Login(BaseCommand): + """ + CTERA Edge Filer Login APIs + """ + + async def login(self, username, password): + host = self._edge.host() + try: + await self._edge.api.form_data('/login', {'username': username, 'password': password}) + logging.getLogger('cterasdk.edge').info("User logged in. %s", {'host': host, 'user': username}) + except CTERAException: + logging.getLogger('cterasdk.edge').error("Login failed. %s", {'host': host, 'user': username}) + raise + + async def logout(self): + """ + Log out of the portal + """ + host = self._edge.host() + user = self._edge.session().account.name + try: + await self._edge.api.form_data('/logout', {'foo': 'bar'}) + logging.getLogger('cterasdk.edge').info("User logged out. %s", {'host': host, 'user': user}) + except CTERAException: + logging.getLogger('cterasdk.edge').error("Logout failed. %s", {'host': host, 'user': user}) + raise diff --git a/cterasdk/audit/postman.py b/cterasdk/audit/postman.py index 8bf784e8..779dc9ad 100644 --- a/cterasdk/audit/postman.py +++ b/cterasdk/audit/postman.py @@ -3,7 +3,7 @@ import atexit import logging import cterasdk.settings -from ..lib import FileSystem +from ..lib.storage import synfs, commonfs from ..common import Object, utf8_decode, utf8_encode from ..convert import fromjsonstr, fromxmlstr, tojsonstr, toxmlstr from ..objects import uri @@ -227,14 +227,13 @@ def __init__(self, raw, protocol, host, port, path, query): @atexit.register def audit(): - if cterasdk.settings.sessions.management.audit.postman.enabled: - fs = FileSystem.instance() + if cterasdk.settings.audit.enabled: collection = Collection.instance() - name = cterasdk.settings.sessions.management.audit.postman.name + name = cterasdk.settings.audit.filename collection.info.name = name if name is not None else str(uuid.uuid4()) filename = f'{collection.info.name}.json' - logging.getLogger('cterasdk.http.trace').info('Saving Postman audit file. %s', { - 'directory': fs.downloads_directory(), + logging.getLogger('cterasdk.http').info('Saving Postman audit file. %s', { + 'directory': commonfs.downloads(), 'name': filename }) - fs.save(fs.downloads_directory(), filename, utf8_encode(collection.serialize())) + synfs.write(commonfs.downloads(), filename, utf8_encode(collection.serialize())) diff --git a/cterasdk/clients/async_requests.py b/cterasdk/clients/async_requests.py index 84a0ba02..452070ce 100644 --- a/cterasdk/clients/async_requests.py +++ b/cterasdk/clients/async_requests.py @@ -9,26 +9,17 @@ logger = logging.getLogger('cterasdk.http') -def session_parameters(client_settings): - return { - 'timeout': aiohttp.ClientTimeout(**client_settings.timeout.kwargs), - 'connector': aiohttp.TCPConnector(**client_settings.connector.kwargs), - 'cookie_jar': aiohttp.CookieJar(**client_settings.cookie_jar.kwargs), - 'trace_configs': client_settings.trace_configs - } - - class Session: """Asynchronous HTTP Session""" - def __init__(self, settings): - self._settings = settings + def __init__(self, **kwargs): + self._kwargs = kwargs self._session = None @property def session(self): if self.closed: - self._session = aiohttp.ClientSession(**session_parameters(self._settings)) + self._session = aiohttp.ClientSession(**self._kwargs) return self._session @property diff --git a/cterasdk/clients/base.py b/cterasdk/clients/base.py index 638b2d2e..238d6c9c 100644 --- a/cterasdk/clients/base.py +++ b/cterasdk/clients/base.py @@ -1,6 +1,7 @@ import logging import threading from . import async_requests +from .settings import ClientSessionSettings, TraceSettings from ..common import utils @@ -64,7 +65,12 @@ def __init__(self, builder=None, session=None, settings=None, authenticator=None self._headers = PersistentHeaders() self._authenticator = authenticator self._builder = builder - self._session = session if session else async_requests.Session(settings) + + default_settings = ClientSessionSettings() + if settings: + default_settings.update(**settings.kwargs) + + self._session = session if session else async_requests.Session(**default_settings, **TraceSettings()) def clone(self, definition, builder=None, authenticator=None): """ diff --git a/cterasdk/clients/clients.py b/cterasdk/clients/clients.py index a640fbef..7d94324f 100644 --- a/cterasdk/clients/clients.py +++ b/cterasdk/clients/clients.py @@ -56,6 +56,34 @@ async def upload(self, path, data, **kwargs): class AsyncWebDAV(AsyncClient): """WebDAV""" + async def download(self, path, **kwargs): + return await super().get(path, **kwargs) + + async def mkcol(self, path): + request = async_requests.MkcolRequest(self._builder(path)) + response = await self.async_request(request) + return await response.text() + + async def copy(self, source, destination, *, overwrite=False): + request = async_requests.CopyRequest(self._builder(source), headers=self._webdav_headers(destination, overwrite)) + response = await self.async_request(request) + return await response.xml() + + async def move(self, source, destination, *, overwrite=False): + request = async_requests.MoveRequest(self._builder(source), headers=self._webdav_headers(destination, overwrite)) + response = await self.async_request(request) + return await response.xml() + + async def delete(self, path): # pylint: disable=arguments-differ + response = await super().delete(path) + return await response.text() + + def _webdav_headers(self, destination, overwrite): + return { + 'Destination': self._builder(destination), + 'Overwrite': 'T' if overwrite is True else 'F' + } + class AsyncJSON(AsyncClient): diff --git a/cterasdk/clients/settings.py b/cterasdk/clients/settings.py index cb78a893..243c246d 100644 --- a/cterasdk/clients/settings.py +++ b/cterasdk/clients/settings.py @@ -1,53 +1,80 @@ -import logging +import copy import cterasdk.settings -from . import tracers -from ..common import Object +from collections.abc import MutableMapping +from aiohttp import TCPConnector, CookieJar, ClientTimeout +from .tracers import requests, session, postman -logger = logging.getLogger('cterasdk.http.trace') +class ClientSessionSettings(MutableMapping): + def __init__(self, *args, **kwargs): + self._mapping = { + 'connector': { + '_classname': TCPConnector, + 'ssl': True + }, + 'timeout': { + '_classname': ClientTimeout, + 'sock_connect': 10, + 'sock_read': 60 + }, + 'cookie_jar': { + '_classname': CookieJar, + 'unsafe': False + } + } + self._mapping.update(dict(*args, **kwargs)) -class ClientSettings(Object): - """ - Asynchronous HTTP Client Session Settings - """ + def update(self, **kwargs): + for k, v in self._mapping.items(): + attributes = kwargs.get(k, None) + self._mapping[k] = attributes + self._mapping[k]['_classname'] = v.get('_classname', None) - def __init__(self, connector=None, timeout=None, cookie_jar=None): - self.connector = connector if connector else TCPConnector(True) - self.timeout = timeout if timeout else ClientTimeout(None, None) - self.cookie_jar = cookie_jar if cookie_jar else CookieJar(False) - self.trace_configs = [ - tracers.logger.trace_config(), - tracers.session.trace_config() - ] - if cterasdk.settings.sessions.management.audit.postman.enabled: - logger.info('Enabling Postman Auditing.') - self.trace_configs.append(tracers.postman.trace_config()) + def __getitem__(self, key): + mapping = copy.deepcopy(self._mapping) + attributes = mapping.get(key, None) + new_instance = attributes.pop('_classname', None) + if new_instance: + return new_instance(**attributes) + return attributes + def __setitem__(self, key, value): + self._mapping[key] = value -class TCPConnector(Object): - """ - Asynchronous HTTP TCP Connector - """ + def __delitem__(self, key): + del self._mapping[key] - def __init__(self, ssl): - self.ssl = ssl + def __iter__(self): + return iter(self._mapping) + def __len__(self): + return len(self._mapping) -class ClientTimeout(Object): - """ - Asynchronous HTTP Client Timeout Settings - """ + def __str__(self): + return str(self._mapping) - def __init__(self, sock_connect, sock_read): - self.sock_connect = sock_connect - self.sock_read = sock_read +class TraceSettings(MutableMapping): -class CookieJar(Object): - """ - Asynchronous HTTP Cookie Jar Settings - """ + def __init__(self): + self._mapping = { + 'trace_configs': [requests.tracer(), session.tracer()] + } + if cterasdk.settings.audit.enabled: + self._mapping['trace_configs'].append(postman.tracer()) - def __init__(self, allow_unsafe): - self.unsafe = allow_unsafe + def __getitem__(self, key): + return self._mapping.get(key, None) + + def __setitem__(self, key, value): + return super().__setitem__(key, value) + + def __delitem__(self, key): + return super().__delitem__(key) + + def __len__(self): + return super().__len__() + + def __iter__(self): + return iter(self._mapping) diff --git a/cterasdk/clients/tracers/__init__.py b/cterasdk/clients/tracers/__init__.py index c5dd9783..d2bff2db 100644 --- a/cterasdk/clients/tracers/__init__.py +++ b/cterasdk/clients/tracers/__init__.py @@ -1 +1 @@ -from . import logger, session, postman # noqa: E402, F401 +from . import requests, session, postman # noqa: E402, F401 diff --git a/cterasdk/clients/tracers/postman.py b/cterasdk/clients/tracers/postman.py index 43753471..6d233d3c 100644 --- a/cterasdk/clients/tracers/postman.py +++ b/cterasdk/clients/tracers/postman.py @@ -2,7 +2,7 @@ from ...audit import postman -def trace_config(): +def tracer(): async def on_request_start(session, ctx, params): # pylint: disable=unused-argument @@ -50,10 +50,10 @@ async def on_request_end(session, ctx, params): ctx.request.request_body(body) postman.Collection.instance().add(ctx.request) - tracer = aiohttp.TraceConfig() - tracer.on_request_start.append(on_request_start) - tracer.on_request_headers_sent.append(on_request_headers_sent) - tracer.on_request_chunk_sent.append(on_request_chunk_sent) - tracer.on_request_end.append(on_request_end) + conf = aiohttp.TraceConfig() + conf.on_request_start.append(on_request_start) + conf.on_request_headers_sent.append(on_request_headers_sent) + conf.on_request_chunk_sent.append(on_request_chunk_sent) + conf.on_request_end.append(on_request_end) - return tracer + return conf diff --git a/cterasdk/clients/tracers/logger.py b/cterasdk/clients/tracers/requests.py similarity index 70% rename from cterasdk/clients/tracers/logger.py rename to cterasdk/clients/tracers/requests.py index 7efaeda9..35d3cc40 100644 --- a/cterasdk/clients/tracers/logger.py +++ b/cterasdk/clients/tracers/requests.py @@ -3,7 +3,10 @@ import aiohttp -def trace_config(): +logger = logging.getLogger('cterasdk.http') + + +def tracer(): async def on_request_start(session, context, params): # pylint: disable=unused-argument @@ -14,7 +17,7 @@ async def on_request_start(session, context, params): 'headers': [dict(params.headers)] if params.headers else [] } } - logging.getLogger('cterasdk.http.trace').debug('Starting request. %s', serialize(param)) + logger.debug('Starting request. %s', serialize(param)) async def on_request_redirect(session, context, params): # pylint: disable=unused-argument @@ -24,7 +27,7 @@ async def on_request_redirect(session, context, params): 'destination': params.response.headers['Location'] } } - logging.getLogger('cterasdk.http.trace').debug('Starting redirect. %s', serialize(param)) + logger.debug('Starting redirect. %s', serialize(param)) async def on_request_end(session, context, params): # pylint: disable=unused-argument @@ -40,14 +43,14 @@ async def on_request_end(session, context, params): 'headers': [dict(params.response.headers)] } } - logging.getLogger('cterasdk.http.trace').debug('Ended request. %s', serialize(param)) + logger.debug('Ended request. %s', serialize(param)) - tracer = aiohttp.TraceConfig() - tracer.on_request_start.append(on_request_start) - tracer.on_request_redirect.append(on_request_redirect) - tracer.on_request_end.append(on_request_end) + conf = aiohttp.TraceConfig() + conf.on_request_start.append(on_request_start) + conf.on_request_redirect.append(on_request_redirect) + conf.on_request_end.append(on_request_end) - return tracer + return conf def serialize(param): diff --git a/cterasdk/clients/tracers/session.py b/cterasdk/clients/tracers/session.py index 9db691f4..ee1d56b4 100644 --- a/cterasdk/clients/tracers/session.py +++ b/cterasdk/clients/tracers/session.py @@ -2,7 +2,7 @@ from ...exceptions import SessionExpired -def trace_config(): +def tracer(): async def on_request_redirect(session, context, params): # pylint: disable=unused-argument @@ -11,6 +11,6 @@ async def on_request_redirect(session, context, params): if location in ['/ServicesPortal/login.html'] or location.startswith(('https://login.microsoftonline.com')): raise SessionExpired() - tracer = aiohttp.TraceConfig() - tracer.on_request_redirect.append(on_request_redirect) - return tracer + conf = aiohttp.TraceConfig() + conf.on_request_redirect.append(on_request_redirect) + return conf diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index 66c582f3..a7468f66 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -12,6 +12,9 @@ def kwargs(self): def __str__(self): return json.dumps(self, default=lambda o: o.__dict__, indent=5) + def __repr__(self): + return str(self) + class Device(Object): diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index 6e09c265..8be6ff53 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -268,7 +268,7 @@ def device_config(self, device, destination=None): File destination, if it is a directory, the original filename will be kept, defaults to the default directory """ try: - destination = destination if destination is not None else f'{cterasdk.settings.downloads.location}/{device}.xml' + destination = destination if destination is not None else f'{commonfs.downloads()}/{device}.xml' return self.download(f'backups/{device}/Device Configuration/db.xml', destination) except CTERAException as error: logging.getLogger('cterasdk.core').error('Failed downloading configuration file. %s', diff --git a/cterasdk/core/types.py b/cterasdk/core/types.py index 655c5f3a..aa3b24b9 100644 --- a/cterasdk/core/types.py +++ b/cterasdk/core/types.py @@ -1,7 +1,7 @@ from abc import ABC from collections import namedtuple from ..common import DateTimeUtils, StringCriteriaBuilder, PredefinedListCriteriaBuilder, CustomListCriteriaBuilder, Object -from ..lib import FileSystem +from ..lib.storage import commonfs from .enum import PortalAccountType, CollaboratorType, FileAccessMode, PlanCriteria, TemplateCriteria, \ BucketType, LocationType, Platform, RetentionMode, Duration, ExtendedAttributes @@ -495,7 +495,7 @@ def after_backup(self, after_backup): @staticmethod def _get_contents(shell_script): - if FileSystem.instance().exists(shell_script): + if commonfs.exists(shell_script): with open(shell_script, 'r', encoding='utf-8') as f: data = f.read() return data diff --git a/cterasdk/direct/client.py b/cterasdk/direct/client.py index ff3f6aad..0933e3a7 100644 --- a/cterasdk/direct/client.py +++ b/cterasdk/direct/client.py @@ -8,17 +8,9 @@ from .stream import Streamer from ..objects.endpoints import DefaultBuilder, EndpointBuilder -from ..clients.settings import ClientSettings, ClientTimeout, TCPConnector from ..clients.clients import AsyncClient, AsyncJSON -def client_settings(parameters): - return ClientSettings( - TCPConnector(parameters.ssl), - ClientTimeout(**parameters.timeout.kwargs) - ) - - class Client: def __init__(self, baseurl, credentials): @@ -26,12 +18,9 @@ def __init__(self, baseurl, credentials): :param str baseurl: Portal URL :param cterasdk.direct.credentials.BaseCredentials credentials: Credentials object """ - self._api = AsyncJSON(EndpointBuilder.new(baseurl, '/directio'), - settings=client_settings(cterasdk.settings.sessions.ctera_direct.api), + self._api = AsyncJSON(EndpointBuilder.new(baseurl, '/directio'), settings=cterasdk.settings.io.direct.api.settings, authenticator=lambda *_: True) - self._client = AsyncClient(DefaultBuilder(), - settings=client_settings(cterasdk.settings.sessions.ctera_direct.storage), - authenticator=lambda *_: True) + self._client = AsyncClient(DefaultBuilder(), settings=cterasdk.settings.io.direct.storage.settings, authenticator=lambda *_: True) self._credentials = credentials async def _direct(self, file_id): diff --git a/cterasdk/edge/services.py b/cterasdk/edge/services.py index 7b9c121d..c81192ef 100644 --- a/cterasdk/edge/services.py +++ b/cterasdk/edge/services.py @@ -180,10 +180,10 @@ def _handle_untrusted_cert(self, server, obj): try: if obj.rc in Services._UNTRUSTED_CERTIFICATE_ERRORS: proceed = False - if cterasdk.settings.sessions.management.edge.services.ssl == 'prompt': + if cterasdk.settings.edge.syn.services.ssl == 'prompt': logging.getLogger('cterasdk.edge').warning(msg=obj.msg) proceed = ask(f"Connect {self._edge.host()} to {server}?") - if cterasdk.settings.sessions.management.edge.services.ssl is False or proceed: + if cterasdk.settings.edge.syn.services.ssl is False or proceed: self._trust_cert[server] = True return self._check_connection(server) except AttributeError: diff --git a/cterasdk/edge/ssh.py b/cterasdk/edge/ssh.py index d2de373e..bb66990a 100644 --- a/cterasdk/edge/ssh.py +++ b/cterasdk/edge/ssh.py @@ -1,5 +1,6 @@ import logging -from ..lib import FileSystem, CryptoServices +from ..lib import CryptoServices +from ..lib.storage import commonfs from ..common import Object from .base_command import BaseCommand @@ -22,7 +23,7 @@ def enable(self, public_key=None, public_key_file=None, exponent=65537, key_size if public_key is None: if public_key_file is not None: - FileSystem.instance().properties(public_key_file) + commonfs.properties(public_key_file) with open(public_key_file, 'r', encoding='utf-8') as f: public_key = f.read() else: diff --git a/cterasdk/edge/support.py b/cterasdk/edge/support.py index 955d57b1..9b552be3 100644 --- a/cterasdk/edge/support.py +++ b/cterasdk/edge/support.py @@ -1,8 +1,7 @@ from datetime import datetime import logging -import cterasdk.settings -from ..lib import FileSystem +from ..lib.storage import synfs, commonfs from ..exceptions import InputError from .base_command import BaseCommand @@ -62,5 +61,6 @@ def get_support_report(self): filename = 'Support-' + self._edge.host() + datetime.now().strftime('_%Y-%m-%dT%H_%M_%S') + '.zip' logging.getLogger('cterasdk.edge').info('Downloading support report. %s', {'host': self._edge.host()}) handle = self._edge.api.handle('/supportreport') - filepath = FileSystem.instance().save(cterasdk.settings.downloads.location, filename, handle) + filepath = synfs.write(commonfs.downloads(), filename, handle) logging.getLogger('cterasdk.edge').info('Support report downloaded. %s', {'filepath': filepath}) + return filepath diff --git a/cterasdk/lib/__init__.py b/cterasdk/lib/__init__.py index a2a18ac2..c5696117 100644 --- a/cterasdk/lib/__init__.py +++ b/cterasdk/lib/__init__.py @@ -4,6 +4,5 @@ from .version import Version # noqa: E402, F401 from .iterator import QueryIterator, BaseResponse, FetchResourcesResponse, \ DefaultResponse, KeyValueQueryIterator, QueryLogsResponse, CursorResponse # noqa: E402, F401 -from .filesystem import FileSystem # noqa: E402, F401 from .tracker import track, ErrorStatus # noqa: E402, F401 from .crypto import CryptoServices, X509Certificate, PrivateKey, create_certificate_chain # noqa: E402, F401 diff --git a/cterasdk/lib/crypto.py b/cterasdk/lib/crypto.py index 1ed4e7b9..10514ca6 100644 --- a/cterasdk/lib/crypto.py +++ b/cterasdk/lib/crypto.py @@ -8,9 +8,11 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, PublicFormat, NoEncryption, load_pem_private_key -import cterasdk.settings from ..exceptions import CTERAException -from .filesystem import FileSystem +from .storage import synfs, commonfs + + +logger = logging.getLogger('cterasdk.crypto') class RSAKeyPair: @@ -28,16 +30,14 @@ def private_key(self): return self._private_key.private_bytes(Encoding.PEM, PrivateFormat.OpenSSH, NoEncryption()) def save(self, dirpath, key_filename): - filesystem = FileSystem.instance() - - logging.getLogger('cterasdk.crypto').info('Saving private key.') - path = filesystem.save(dirpath, f'{key_filename}.pem', self.private_key) + logger.info('Saving private key.') + path = synfs.write(dirpath, f'{key_filename}.pem', self.private_key) os.chmod(path, 0o600) - logging.getLogger('cterasdk.crypto').info('Saved private key. %s', {'filepath': path, 'format': 'PEM'}) + logger.info('Saved private key. %s', {'filepath': path, 'format': 'PEM'}) - logging.getLogger('cterasdk.crypto').info('Saving public key.') - path = filesystem.save(dirpath, f'{key_filename}.pub', self.public_key) - logging.getLogger('cterasdk.crypto').info('Saved public key. %s', {'filepath': path, 'format': 'OpenSSH'}) + logger.info('Saving public key.') + path = synfs.write(dirpath, f'{key_filename}.pub', self.public_key) + logger.info('Saved public key. %s', {'filepath': path, 'format': 'OpenSSH'}) class CryptoServices: @@ -52,7 +52,7 @@ def generate_rsa_key_pair(exponent=65537, key_size=2048): def generate_and_save_key_pair(key_filename, exponent=65537, key_size=2048, dirpath=None): key_pair = CryptoServices.generate_rsa_key_pair(exponent, key_size) if not dirpath: - dirpath = cterasdk.settings.downloads.location + dirpath = commonfs.downloads() key_pair.save(dirpath, key_filename) return key_pair.public_key.decode('utf-8') @@ -98,11 +98,11 @@ def load_private_key(key, password=None): if isinstance(key, bytes): return PrivateKey.from_bytes(key, password) - if FileSystem.instance().exists(key): + if commonfs.exists(key): return PrivateKey.from_file(key, password) return PrivateKey.from_string(key, password) except ValueError as e: - logging.getLogger('cterasdk.crypto').error('Failed loading private key.') + logger.error('Failed loading private key.') raise CTERAException('Failed loading private key', e, reason=str(e)) @@ -157,11 +157,11 @@ def load_certificate(cert): if isinstance(cert, bytes): return X509Certificate.from_bytes(cert) - if FileSystem.instance().exists(cert): + if commonfs.exists(cert): return X509Certificate.from_file(cert) return X509Certificate.from_string(cert) except ValueError as e: - logging.getLogger('cterasdk.crypto').error('Failed loading certificate.') + logger.error('Failed loading certificate.') raise CTERAException('Failed loading certificate', e, reason=str(e)) def __str__(self): diff --git a/cterasdk/lib/filesystem.py b/cterasdk/lib/filesystem.py deleted file mode 100644 index ee0b297a..00000000 --- a/cterasdk/lib/filesystem.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import errno -import mimetypes -import logging -from pathlib import Path - -import aiofiles -import cterasdk.settings - - -class FileSystem: # pylint: disable=unused-private-member - - __instance = None - - def __init__(self): - if FileSystem.__instance is not None: - raise Exception("FileSystem is a Singleton Class.") - FileSystem.__instance = self - - @staticmethod - def instance(): - if FileSystem.__instance is None: - FileSystem() - return FileSystem.__instance - - @staticmethod - def expanduser(p): - """ - Return a new path with expanded ~ and ~user constructs - - :param str p: Path - :returns: Absolute Path. - :rtype: str - """ - return Path(p).expanduser() - - @staticmethod - def exists(p): - """ - Check if a file or a directory exists - - :param str p: Path - :returns: ``True`` if exists, ``False`` otherwise. - :rtype: bool - """ - return Path(p).exists() - - def rename(self, parent, source, destination): # pylint: disable=no-self-use - """ - Rename a file or a directory. - - :param str parent: Parent directory - :param str source: Source file or directory name - :param str destination: Destination file or directory name - :returns: Parent directory, and file or directory name - :rtype: tuple(str, str) - """ - source = Path(parent).joinpath(source) - if not source.exists(): - logging.getLogger('cterasdk.filesystem').error('Rename failed. File not found. %s', {'path': source.as_posix()}) - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), source.as_posix()) - destination = Path(parent).joinpath(destination) - source.rename(destination) - return destination.parent.as_posix(), destination.name - - @staticmethod - def join(*paths): - return Path(*paths) - - def is_dir(self, p): - """ - Check is a directory. - - :param str p: Path - :returns: ``True`` if a directory, ``False`` otherwise. - :rtype: bool - """ - p = self.expanduser(p) - return p.is_dir() - - def downloads_directory(self): - """ - Get downloads directory. - - :returns: Directory Path - :rtype: str - """ - location = cterasdk.settings.downloads.location - if not self.is_dir(location): - logging.getLogger('cterasdk.filesystem').error('Could not find downloads directory. %s', {'path': location}) - raise FileNotFoundError(errno.ENOENT, 'No such directory', location) - return location - - def split_file_directory(self, location): - """ - Split file and directory. - - :param str path: Path - - Returns: - 1. (parent directory, file name), if a file exists - 2. (parent directory, file name), if a directory exists - 3. (parent directory, file name), if the parent directory exists - 4. Raises ``FileNotFoundError`` if neither the object nor the parent directory exist - """ - p = self.expanduser(location) - if p.exists(): - if p.is_dir(): - filename = None - else: - filename = p.name - p = p.parent - elif p.parent.exists(): - filename = p.name - p = p.parent - else: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), location) - return str(p.resolve()), filename - - def generate_file_location(self, location=None, default_filename=None): - """ - Compute destination file path. - - :param str location: Path to a file or a folder - :param str default_filename: Default file name, unless ``location`` already specifies a file path - :returns: Tuple including the destination directory and file name - :rtype: tuple(str, str) - """ - parent = filename = None - if location: - parent, filename = self.split_file_directory(location) - else: - parent = self.downloads_directory() - - if not filename: - filename = default_filename - - return (parent, filename) - - def properties(self, location): - """ - Get file properties. - - :param str location: Path - :returns: File name, size and type - :rtype: dict - """ - p = self.expanduser(location) - - if not p.exists(): - logging.getLogger('cterasdk.filesystem').error('File not found. %s', {'path': p.as_posix()}) - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), p.as_posix()) - - if not p.is_file(): - logging.getLogger('cterasdk.filesystem').error('No such file. %s', {'path': p.as_posix()}) - raise FileNotFoundError(errno.ENOENT, 'Not such file', p.as_posix()) - - return dict(name=p.name, size=str(p.stat().st_size), mimetype=mimetypes.guess_type(location)) - - @staticmethod - def file_version(filename, version): - """ - Append version number to file name. - - :param str filename: File name - :param int version: File version - :returns: File name appended with a version number - :rtype: str - """ - idx = filename.rfind('.') - extension = '' - if idx > 0: - name = filename[:idx] - extension = filename[idx:] - else: - name = filename - return f'{name} ({str(version)}){extension}' - - @staticmethod - def compute_zip_file_name(cloud_directory, files): - """ - Compute zip file name. - """ - if len(files) > 1: - path = Path(cloud_directory) - else: - path = Path(files[0]) - return f'{path.stem}.zip' - - def _before_save(self, directory, filename): - """ - Check directory exists, and compute temporary file name. - """ - parent = self.expanduser(directory) - if not parent.exists(): - raise FileNotFoundError(errno.ENOENT, 'No such directory', directory) - - temporary_file = parent.joinpath(f'{filename}.ctera') - return (parent, temporary_file) - - def _after_write(self, parent, temporary_file, filename): - """ - Move file to its final destination. - """ - origin = filename - version = 0 - while True: - try: - self.rename(parent, temporary_file.as_posix(), filename) - break - except (FileExistsError, IsADirectoryError): - logging.getLogger('cterasdk.filesystem').debug('File exists. %s', {'path': parent.as_posix(), 'name': filename}) - version = version + 1 - filename = self.file_version(origin, version) - - filepath = parent.joinpath(filename) - logging.getLogger('cterasdk.filesystem').info('Saved. %s', {'path': filepath.as_posix()}) - return filepath.as_posix() - - def save(self, directory, filename, handle): - """ - Save file. - - :param str parent: Directory path. - :param str filename: File name. - :param object handle: File handle. - :returns: File path - :rtype: str - """ - parent, temporary_file = self._before_save(directory, filename) # Check directory exists, and compute temporary file name. - self.write(temporary_file, handle) # Write temporary file. - return self._after_write(parent, temporary_file, filename) # Rename to destination - - @staticmethod - def write(p, handle): - with open(p, 'w+b') as fd: - if isinstance(handle, bytes): - fd.write(handle) - else: - for chunk in handle.iter_content(chunk_size=8192): - fd.write(chunk) - logging.getLogger('cterasdk.filesystem').debug('Write Complete. %s', {'path': p}) - - @staticmethod - async def async_write(p, handle): - async with aiofiles.open(p, 'w+b') as fd: - async for chunk in handle.async_iter_content(chunk_size=8192): - await fd.write(chunk) - logging.getLogger('cterasdk.filesystem').debug('Write Complete. %s', {'path': p}) diff --git a/cterasdk/lib/storage/commonfs.py b/cterasdk/lib/storage/commonfs.py index 83c8a226..689fa920 100644 --- a/cterasdk/lib/storage/commonfs.py +++ b/cterasdk/lib/storage/commonfs.py @@ -83,7 +83,7 @@ def downloads(): :returns: Directory path :rtype: Path object """ - directory = expanduser(cterasdk.settings.downloads.location) + directory = expanduser(cterasdk.settings.io.downloads) if not is_dir(directory): logger.error('Directory not found: %s', directory) raise FileNotFoundError(errno.ENOENT, 'Directory not found', directory) diff --git a/cterasdk/logging.py b/cterasdk/logging.py deleted file mode 100644 index b30b73ed..00000000 --- a/cterasdk/logging.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import sys -import logging -import cterasdk.settings - -parameters = { - 'format': cterasdk.settings.logging.format, - 'datefmt': cterasdk.settings.logging.date_format, -} - -filename = os.environ.get('cterasdk.log') -if filename: - parameters['filename'] = filename -else: - parameters['stream'] = sys.stdout - -logging.basicConfig(**parameters) - -for logger_conf in cterasdk.settings.logging.loggers: - logger = logging.getLogger(f'{__package__}.{logger_conf.name}') - logger.setLevel(logging.getLevelName(logger_conf.level.upper())) diff --git a/cterasdk/objects/__init__.py b/cterasdk/objects/__init__.py index f6760514..66148a2d 100644 --- a/cterasdk/objects/__init__.py +++ b/cterasdk/objects/__init__.py @@ -2,3 +2,4 @@ from .synchronous.edge import Edge # noqa: E402, F401 from .synchronous.drive import Drive # noqa: E402, F401 from .asynchronous.core import AsyncGlobalAdmin, AsyncServicesPortal # noqa: E402, F401 +from .asynchronous.edge import AsyncEdge # noqa: E402, F401 diff --git a/cterasdk/objects/asynchronous/core.py b/cterasdk/objects/asynchronous/core.py index df3be025..94a2f78c 100644 --- a/cterasdk/objects/asynchronous/core.py +++ b/cterasdk/objects/asynchronous/core.py @@ -1,19 +1,10 @@ import cterasdk.settings -from ..services import CTERA, client_settings +from ..services import AsyncManagement from ..endpoints import EndpointBuilder from ...clients import clients - from .. import authenticators - from ...lib.session.core import Session - -from ...asynchronous.core import files -from ...asynchronous.core import login -from ...asynchronous.core import cloudfs -from ...asynchronous.core import notifications -from ...asynchronous.core import portals -from ...asynchronous.core import settings -from ...asynchronous.core import users +from ...asynchronous.core import files, login, cloudfs, notifications, portals, settings, users class Clients: @@ -61,29 +52,18 @@ def builder(self): return self._webdav._builder # pylint: disable=protected-access -class AsyncPortal(CTERA): - - async def __aenter__(self): - return self +class AsyncPortal(AsyncManagement): def __init__(self, host, port=None, https=True): - super().__init__(host, port, https, base=None) - self._default = clients.AsyncClient(EndpointBuilder.new(self.base), - settings=client_settings(cterasdk.settings.sessions.metadata_connector), - authenticator=self._authenticator) + super().__init__(host, port, https, None, cterasdk.settings.core.asyn.settings) self._ctera_session = Session(self.host(), self.context) self._ctera_clients = Clients(self) - self.cloudfs = cloudfs.CloudFS(self) self.files = files.CloudDrive(self) self.notifications = notifications.Notifications(self) self.settings = settings.Settings(self) self.users = users.Users(self) - @property - def default(self): - return self._default - @property def v1(self): return self.clients.v1 @@ -96,18 +76,6 @@ def v2(self): def io(self): return self.clients.io - async def login(self, username, password): - self._before_login() - await self._login_object.login(username, password) - await self._ctera_session.async_start_session(self) - self._after_login() - - async def logout(self): - if self._ctera_session.connected: - await self._login_object.logout() - self._ctera_session.stop_session() - await self.default.close() - @property def _login_object(self): return login.Login(self) @@ -115,9 +83,6 @@ def _login_object(self): def _authenticator(self, url): return authenticators.core(self.session(), url, self.context) - async def __aexit__(self, exc_type, exc, tb): - await self.default.close() - class AsyncGlobalAdmin(AsyncPortal): diff --git a/cterasdk/objects/asynchronous/edge.py b/cterasdk/objects/asynchronous/edge.py new file mode 100644 index 00000000..1c5ba050 --- /dev/null +++ b/cterasdk/objects/asynchronous/edge.py @@ -0,0 +1,77 @@ +import cterasdk.settings +from ..services import AsyncManagement +from ..endpoints import EndpointBuilder +from ...clients import clients +from .. import authenticators +from ...lib.session.edge import Session +from ...asynchronous.edge import login, files + + +class Clients: + + def __init__(self, edge): + self.api = edge.default.clone(clients.AsyncAPI, EndpointBuilder.new(edge.base, '/admingui/api')) + self.io = IO(edge) + + +class IO: + + def __init__(self, edge): + self._edge = edge + self._webdav = edge.default.clone(clients.AsyncWebDAV, EndpointBuilder.new(edge.base, '/localFiles')) + + @property + def download(self): + return self._webdav.download + + @property + def download_zip(self): + return self._edge.default.form_data # pylint: disable=protected-access + + @property + def upload(self): + return self._edge.default.form_data # pylint: disable=protected-access + + @property + def mkdir(self): + return self._webdav.mkcol + + @property + def copy(self): + return self._webdav.copy + + @property + def move(self): + return self._webdav.move + + @property + def delete(self): + return self._webdav.delete + + +class AsyncEdge(AsyncManagement): + + def __init__(self, host=None, port=None, https=True, *, base=None): + super().__init__(host, port, https, base, cterasdk.settings.edge.asyn.settings) + self._ctera_session = Session(self.host()) + self._ctera_clients = Clients(self) + self.files = files.FileBrowser(self) + + @property + def v1(self): + return self.clients.v1 + + @property + def api(self): + return self.clients.api + + @property + def io(self): + return self.clients.io + + @property + def _login_object(self): + return login.Login(self) + + def _authenticator(self, url): + return authenticators.edge(self.session(), url) \ No newline at end of file diff --git a/cterasdk/objects/services.py b/cterasdk/objects/services.py index 2d160c49..05fcb871 100644 --- a/cterasdk/objects/services.py +++ b/cterasdk/objects/services.py @@ -1,9 +1,7 @@ from abc import abstractmethod -import cterasdk.settings from . import endpoints, uri from .utils import URI -from ..clients.settings import ClientSettings, ClientTimeout, TCPConnector, CookieJar from ..clients import clients from ..common import Object from ..convert import tojsonstr @@ -76,24 +74,42 @@ def whoami(self): return self._ctera_session.whoami() -def client_settings(parameters): - return ClientSettings( - TCPConnector(parameters.ssl), - ClientTimeout(**parameters.timeout.kwargs), - CookieJar(parameters.allow_unsafe) - ) +class AsyncManagement(CTERA): + async def __aenter__(self): + return self + + def __init__(self, host, port, https, base, settings): + super().__init__(host, port, https, base) + self._default = clients.AsyncClient(endpoints.EndpointBuilder.new(self.base), settings=settings, authenticator=self._authenticator) + + @property + def default(self): + return self._default + + async def login(self, username, password): + self._before_login() + await self._login_object.login(username, password) + await self._ctera_session.async_start_session(self) + self._after_login() + + async def logout(self): + if self._ctera_session.connected: + await self._login_object.logout() + self._ctera_session.stop_session() + await self.default.close() + + async def __aexit__(self, exc_type, exc, tb): + await self.default.close() class Management(CTERA): def __enter__(self): return self - def __init__(self, host, port, https, base): + def __init__(self, host, port, https, base, settings): super().__init__(host, port, https, base) - self._default = clients.Client(endpoints.EndpointBuilder.new(self.base), - settings=client_settings(cterasdk.settings.sessions.management), - authenticator=self._authenticator) + self._default = clients.Client(endpoints.EndpointBuilder.new(self.base), settings=settings, authenticator=self._authenticator) def login(self, username, password): """ diff --git a/cterasdk/objects/synchronous/core.py b/cterasdk/objects/synchronous/core.py index 1fe1b45a..55dc53a3 100644 --- a/cterasdk/objects/synchronous/core.py +++ b/cterasdk/objects/synchronous/core.py @@ -1,46 +1,16 @@ from abc import abstractmethod +import cterasdk.settings from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder - from .. import authenticators - from ...lib.session.core import Session - -from ...core import activation -from ...core import admins -from ...core import antivirus -from ...core import buckets -from ...core import cli -from ...core import cloudfs -from ...core import connection -from ...core import credentials -from ...core import devices -from ...core import directoryservice -from ...core import domains -from ...core import files -from ...core import firmwares -from ...core import groups -from ...core import kms -from ...core import licenses -from ...core import login -from ...core import logs -from ...core import mail -from ...core import messaging -from ...core import plans -from ...core import portals -from ...core import reports -from ...core import roles -from ...core import servers -from ...core import settings -from ...core import setup -from ...core import ssl -from ...core import startup -from ...core import storage_classes -from ...core import syslog -from ...core import taskmgr -from ...core import templates -from ...core import users +from ...core import ( + activation, admins, antivirus, buckets, cli, cloudfs, connection, credentials, + devices, directoryservice, domains, files, firmwares, groups, kms, licenses, + login, logs, mail, messaging, plans, portals, reports, roles, servers, settings, + setup, ssl, startup, storage_classes, syslog, taskmgr, templates, users, +) class Clients: @@ -78,11 +48,9 @@ def builder(self): class Portal(Management): # pylint: disable=too-many-instance-attributes def __init__(self, host, port=None, https=True): - super().__init__(host, port, https, base=None) - + super().__init__(host, port, https, None, cterasdk.settings.core.syn.settings) self._ctera_session = Session(self.host(), self.context) self._ctera_clients = Clients(self) - self.activation = activation.Activation(self) self.admins = admins.Administrators(self) self.backups = files.Backups(self) diff --git a/cterasdk/objects/synchronous/drive.py b/cterasdk/objects/synchronous/drive.py index 8f8a3306..2ab59a78 100644 --- a/cterasdk/objects/synchronous/drive.py +++ b/cterasdk/objects/synchronous/drive.py @@ -1,15 +1,9 @@ +import cterasdk.settings from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder - from ...lib.session.edge import Session - -from ...edge import backup -from ...edge import cli -from ...edge import logs -from ...edge import services -from ...edge import support -from ...edge import sync +from ...edge import backup, cli, logs, services, support, sync class Clients: @@ -27,10 +21,9 @@ def __init__(self, drive, Portal): class Drive(Management): def __init__(self, host=None, port=None, https=True, Portal=None, *, base=None): - super().__init__(host, port, https, base=base) + super().__init__(host, port, https, base, cterasdk.settings.drive.syn.settings) self._ctera_session = Session(self.host()) self._ctera_clients = Clients(self, Portal) - self.backup = backup.Backup(self) self.cli = cli.CLI(self) self.logs = logs.Logs(self) diff --git a/cterasdk/objects/synchronous/edge.py b/cterasdk/objects/synchronous/edge.py index 02711f5b..9d706ecc 100644 --- a/cterasdk/objects/synchronous/edge.py +++ b/cterasdk/objects/synchronous/edge.py @@ -1,54 +1,17 @@ +import cterasdk.settings from ...clients import clients from ..services import Management from ..endpoints import EndpointBuilder - from .. import authenticators - from ...lib.session.edge import Session -from ...edge import afp -from ...edge import aio -from ...edge import array -from ...edge import audit -from ...edge import backup -from ...edge import cache -from ...edge import cli -from ...edge import config -from ...edge import connection -from ...edge import ctera_migrate -from ...edge import dedup -from ...edge import directoryservice -from ...edge import drive -from ...edge import files -from ...edge import firmware -from ...edge import ftp -from ...edge import groups -from ...edge import licenses -from ...edge import login -from ...edge import logs -from ...edge import mail -from ...edge import network -from ...edge import nfs -from ...edge import ntp -from ...edge import power -from ...edge import remote -from ...edge import rsync -from ...edge import ransom_protect -from ...edge import services -from ...edge import shares -from ...edge import shell -from ...edge import smb -from ...edge import snmp -from ...edge import ssh -from ...edge import ssl -from ...edge import support -from ...edge import sync -from ...edge import syslog -from ...edge import taskmgr -from ...edge import telnet -from ...edge import timezone -from ...edge import users -from ...edge import volumes +from ...edge import ( + afp, aio, array, audit, backup, cache, cli, config, connection, ctera_migrate, + dedup, directoryservice, drive, files, firmware, ftp, groups, licenses, login, + logs, mail, network, nfs, ntp, power, remote, rsync, ransom_protect, services, + shares, shell, smb, snmp, ssh, ssl, support, sync, syslog, taskmgr, telnet, + timezone, users, volumes, +) class Clients: @@ -103,10 +66,9 @@ def delete(self): class Edge(Management): # pylint: disable=too-many-instance-attributes def __init__(self, host=None, port=None, https=True, Portal=None, *, base=None): - super().__init__(host, port, https, base=base) + super().__init__(host, port, https, base, cterasdk.settings.edge.syn.settings) self._ctera_session = Session(self.host()) self._ctera_clients = Clients(self, Portal) - self.afp = afp.AFP(self) self.aio = aio.AIO(self) self.array = array.Array(self) diff --git a/cterasdk/settings.py b/cterasdk/settings.py index dc95f4f8..2c21aaf3 100644 --- a/cterasdk/settings.py +++ b/cterasdk/settings.py @@ -1,22 +1,25 @@ -from pathlib import Path import yaml -from .convert import tojsonstr, fromjsonstr +import json +import logging +from pathlib import Path +from .convert import fromjsonstr +from .lib.storage import commonfs + +logger = logging.getLogger('cterasdk') -sessions, downloads, logging = None, None, None -def default_settings(): - global sessions, downloads, logging # # pylint: disable=global-statement - with open(Path(__file__).parent.absolute().joinpath('settings.yml'), 'r', encoding='utf-8') as f: - sdk_settings = yaml.safe_load(f) - sessions = convert_to_object(sdk_settings['sessions']) - downloads = convert_to_object(sdk_settings['downloads']) - logging = convert_to_object(sdk_settings['logging']) +settings = Path(__file__).parent.absolute().joinpath('settings.yml') +try: + commonfs.properties(settings) +except FileNotFoundError: + logger.fatal("Configuration file 'settings.yml' not found. Please check your installation and try again.") + raise -def convert_to_object(data): - return fromjsonstr(tojsonstr(data)) +with open(settings, 'r', encoding='utf-8') as f: + settings = fromjsonstr(json.dumps(yaml.safe_load(f))) -default_settings() +core, edge, io, audit = settings.core, settings.edge, settings.io, settings.audit diff --git a/cterasdk/settings.yml b/cterasdk/settings.yml index a703d1cd..6929970e 100644 --- a/cterasdk/settings.yml +++ b/cterasdk/settings.yml @@ -1,58 +1,60 @@ --- -sessions: - management: - allow_unsafe: true +default_cookie_jar: &cookie_jar + cookie_jar: + unsafe: true + +default_connector: &connector + connector: ssl: true - timeout: - sock_connect: 30 - sock_read: 60 - edge: - services: - ssl: prompt - audit: - postman: - name: ~ - enabled: false - ctera_direct: - api: - ssl: true + +default_timeout: &timeout + timeout: + sock_connect: 5 + sock_read: 10 + +default_settings: &default_settings + settings: + <<: *cookie_jar + <<: *connector + <<: *timeout + +# CTERA Portal synchronous and asynchronous client configuration. +core: + syn: + settings: + <<: *cookie_jar + <<: *connector timeout: - sock_connect: 5 - sock_read: 10 + sock_connect: 30 + sock_read: 60 + asyn: + <<: *default_settings + +# CTERA Edge Filer synchronous client configuration. +edge: + syn: + <<: *default_settings + services: + ssl: prompt + asyn: + <<: *default_settings + +# CTERA Drive synchronous client configuration. +drive: + syn: + <<: *default_settings + +# CTERA asynchronous Direct IO client configuration. +io: + direct: + api: + <<: *default_settings storage: - ssl: true - timeout: - sock_connect: 5 - sock_read: 10 + <<: *default_settings streamer: max_workers: 20 - metadata_connector: - allow_unsafe: true - ssl: true - timeout: - sock_connect: 5 - sock_read: 10 -logging: - format: '%(asctime)s,%(msecs)3d %(levelname)7s [%(filename)s:%(lineno)d] [%(funcName)s] - %(message)s' - date_format: '%Y-%m-%d %H:%M:%S' - loggers: - - name: common - level: info - - name: core - level: info - - name: edge - level: info - - name: metadata.connector - level: info - - name: filesystem - level: error - - name: direct - level: error - - name: crypto - level: error - - name: http - level: error - - name: http.trace - level: error -downloads: - location: ~/Downloads \ No newline at end of file + downloads: ~/Downloads + +audit: + enabled: false + filename: ~ \ No newline at end of file diff --git a/docs/source/UserGuides/DataServices/DirectIO.rst b/docs/source/UserGuides/DataServices/DirectIO.rst index b6049f15..1c49c565 100644 --- a/docs/source/UserGuides/DataServices/DirectIO.rst +++ b/docs/source/UserGuides/DataServices/DirectIO.rst @@ -78,8 +78,8 @@ During testing, you may need to disable TLS verification if the Portal or Object import cterasdk.settings - cterasdk.settings.sessions.ctera_direct.api.ssl = False # disable CTERA Portal TLS verification - cterasdk.settings.sessions.ctera_direct.storage.ssl = False # disable Object Storage TLS verification + cterasdk.settings.io.direct.api.settings.connector.ssl = False # disable CTERA Portal TLS verification + cterasdk.settings.io.direct.storage.settings.connector.ssl = False # disable Object Storage TLS verification Blocks API @@ -110,8 +110,8 @@ Blocks API await f.write(block.data) if __name__ == '__main__': - cterasdk.settings.sessions.ctera_direct.api.ssl = False - cterasdk.settings.sessions.ctera_direct.storage.ssl = False + cterasdk.settings.io.direct.api.settings.connector.ssl = False + cterasdk.settings.io.direct.storage.settings.connector.ssl = False file_id = 12345 @@ -174,8 +174,8 @@ Streamer API if __name__ == '__main__': - cterasdk.settings.sessions.ctera_direct.api.ssl = False - cterasdk.settings.sessions.ctera_direct.storage.ssl = False + cterasdk.settings.io.direct.api.settings.connector.ssl = False + cterasdk.settings.io.direct.storage.settings.connector.ssl = False file_id = 12345 diff --git a/docs/source/UserGuides/DataServices/NotificationService.rst b/docs/source/UserGuides/DataServices/NotificationService.rst index e21ce109..8125569a 100644 --- a/docs/source/UserGuides/DataServices/NotificationService.rst +++ b/docs/source/UserGuides/DataServices/NotificationService.rst @@ -101,7 +101,7 @@ Ancestors async def main(): - cterasdk.settings.sessions.metadata_connector.ssl = False + cterasdk.settings.core.asyn.settings.connector.ssl = False cursor = None queue = asyncio.Queue() # Shared queue between producer and consumer threads async with AsyncGlobalAdmin('tenant.ctera.com') as admin: @@ -164,8 +164,8 @@ Code Snippets async def main(): - cterasdk.settings.sessions.metadata_connector.ssl = False - cterasdk.settings.sessions.ctera_direct.api.ssl = False + cterasdk.settings.core.asyn.settings.connector.ssl = False + cterasdk.settings.io.direct.api.settings.connector.ssl = False cursor = None queue = asyncio.Queue() # Shared queue between producer and consumer threads async with AsyncGlobalAdmin('tenant.ctera.com') as admin: diff --git a/docs/source/UserGuides/Edge/Configuration.rst b/docs/source/UserGuides/Edge/Configuration.rst index 0dc3f938..ef5541a9 100644 --- a/docs/source/UserGuides/Edge/Configuration.rst +++ b/docs/source/UserGuides/Edge/Configuration.rst @@ -495,7 +495,7 @@ Connecting to CTERA Portal .. automethod:: cterasdk.edge.services.Services.connect :noindex: -.. warning:: To ignore certificate errors when connecting to CTERA Portal, use: ``cterasdk.settings.sessions.management.edge.services.ssl = False`` +.. warning:: To ignore certificate errors when connecting to CTERA Portal, use: ``cterasdk.settings.edge.syn.services.ssl = False`` .. .. code-block:: python diff --git a/docs/source/UserGuides/Edge/Quickstart.rst b/docs/source/UserGuides/Edge/Quickstart.rst index bd70d0d5..d5cd639d 100644 --- a/docs/source/UserGuides/Edge/Quickstart.rst +++ b/docs/source/UserGuides/Edge/Quickstart.rst @@ -29,14 +29,14 @@ Now, let's login to the CTERA Edge Filer: """By default, the Edge Filer uses a self-signed certificate. Therefore, we have to disable TLS/SSL verification to successfully login for the first time. """ - cterasdk.settings.sessions.management.ssl = False # disables TLS verification + cterasdk.settings.edge.syn.settings.connector.ssl = False # disables TLS verification with Edge('192.168.0.1') as edge: edge.login('admin-username', 'admin-password') if __name__ == '__main__': main() -.. note:: To ignore SSL errors, or log in using an IP address, use: ``cterasdk.settings.sessions.management.ssl = False`` +.. note:: To ignore SSL errors, or log in using an IP address, use: ``cterasdk.settings.edge.syn.settings.connector.ssl = False`` Now we have an authenticated ``edge`` session. We can now proceed to access the Edge Filer API's. @@ -62,7 +62,7 @@ If we combine the two examples above, we get the following result: """By default, the Edge Filer uses a self-signed certificate. Therefore, we have to disable TLS/SSL verification to successfully login for the first time. """ - cterasdk.settings.sessions.management.ssl = False + cterasdk.settings.edge.syn.settings.connector.ssl = False with Edge('192.168.0.1') as edge: edge.login('admin-username', 'admin-password') edge.config.set_hostname('edge-filesystem') @@ -89,7 +89,7 @@ And equivalnent example to the one given above: """By default, the Edge Filer uses a self-signed certificate. Therefore, we have to disable TLS/SSL verification to successfully login for the first time. """ - cterasdk.settings.sessions.management.ssl = False + cterasdk.settings.edge.syn.settings.connector.ssl = False edge = Edge('192.168.0.1') edge.login('admin-username', 'admin-password') diff --git a/docs/source/UserGuides/Miscellaneous/Index.rst b/docs/source/UserGuides/Miscellaneous/Index.rst index c1193d18..0feef095 100644 --- a/docs/source/UserGuides/Miscellaneous/Index.rst +++ b/docs/source/UserGuides/Miscellaneous/Index.rst @@ -74,8 +74,8 @@ By default, auditing is disabled for performance reasons. If you need to enable .. code-block:: python import cterasdk.settings - cterasdk.settings.sessions.management.audit.postman.enabled = True - cterasdk.settings.sessions.management.audit.postman.name = 'name-of-your-postman-collection-file' + cterasdk.settings.audit.enabled = True + cterasdk.settings.audit.filename = 'name-of-your-postman-collection-file' admin = GlobalAdmin('tenant.ctera.com') diff --git a/docs/source/UserGuides/Portal/Quickstart.rst b/docs/source/UserGuides/Portal/Quickstart.rst index eed8d395..dd46a78f 100644 --- a/docs/source/UserGuides/Portal/Quickstart.rst +++ b/docs/source/UserGuides/Portal/Quickstart.rst @@ -45,7 +45,7 @@ Alternatively, we can also login to CTERA Portal as a Team Portal (tenant) user if __name__ == '__main__': main() -.. note:: To ignore SSL errors, or log in using an IP address, use: ``cterasdk.settings.sessions.management.ssl = False`` +.. note:: To ignore SSL errors, or log in using an IP address, use: ``cterasdk.settings.core.syn.settings.connector.ssl = False`` Now we have an authenticated ``admin`` or ``user`` session. We can now proceed to access the Portal API's. @@ -103,9 +103,7 @@ And equivalnent example to the one given above: .. code-block:: python - import cterasdk.settings from cterasdk import Edge - from cterasdk import GlobalAdmin def main(): diff --git a/docs/source/index.rst b/docs/source/index.rst index 35dc13a5..947f2608 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ Edge from cterasdk import Edge def main(): - cterasdk.settings.sessions.management.ssl = False # for unstrusted or self-signed certificates + cterasdk.settings.edge.syn.settings.connector.ssl = False # for unstrusted or self-signed certificates with Edge('192.168.0.1') as edge: edge.login('admin-username', 'admin-password') print('Hostname: ', edge.config.get_hostname()) @@ -74,7 +74,7 @@ Portal from cterasdk import GlobalAdmin def main(): - cterasdk.settings.sessions.management.ssl = False # for unstrusted or self-signed certificates + cterasdk.settings.core.syn.settings.connector.ssl = False # for unstrusted or self-signed certificates with GlobalAdmin('tenant.ctera.com') as admin: admin.login('admin-username', 'admin-password') for user in admin.users.list_local_users(include=['firstName', 'lastName', 'email']): diff --git a/samples/gateway_sample.py b/samples/gateway_sample.py deleted file mode 100644 index 3117f066..00000000 --- a/samples/gateway_sample.py +++ /dev/null @@ -1,148 +0,0 @@ -import logging - -from cterasdk import Edge, edge_enum, config, CTERAException, edge_types - -from sample_base import CTERASDKSampleBase - - -class GatewaySample(CTERASDKSampleBase): - _gateway_address_request_string = "the Gateway Address" - _gateway_username_request_string = "the user name" - _gateway_new_hostname_request_string = "a new hostname for the Gateway" - _gateway_new_location_request_string = "a new location for the Gateway" - _gateway_portal_address_request_string = "the Portal Address" - _gateway_volume_name_request_string = "a name for the Gateway volume" - _share_directory_name_request_string = "a new for the new shared directory" - - def __init__(self): - self._device = None - self._volume_name = None - - def run(self): - print("CTERA SDK - Gateway Demo") - self._connect_to_gateway() - self._device.test() - self._create_first_user() - self._create_user() - self._set_host_name() - self._set_location() - self._set_dns_server() - self._configure_volume() - self._connect_to_portal() - self._start_cloud_caching() - self._disable_first_time_wizard() - self._create_share() - print("Demo Completed") - - def _connect_to_gateway(self): - gateway_address = GatewaySample._get_input(GatewaySample._gateway_address_request_string) - self._device = Edge(gateway_address) - - def _create_first_user(self): - print("Creating the Gateway's first user") - username = GatewaySample._get_input(GatewaySample._gateway_username_request_string) - password = GatewaySample._get_password(username) - self._device.users.add_first_user(username, password) - print("Logged in as {username}".format(username=username)) - - def _create_user(self): - print("Adding a user to the Gateway") - username = GatewaySample._get_input(GatewaySample._gateway_username_request_string) - password = GatewaySample._get_password(username) - self._device.users.add(username, password, full_name=username, email="{username}@example.com".format(username=username)) - print("User {username} was added successfully".format(username=username)) - - def _set_host_name(self): - print("Setting the Gateway's Host Name") - new_host_name = GatewaySample._get_input(GatewaySample._gateway_new_hostname_request_string) - self._device.config.set_hostname(new_host_name) - print("Gateway Host name was set to {new_host_name}".format(new_host_name=new_host_name)) - - def _set_location(self): - print("Setting the Gateway's Location") - new_location = GatewaySample._get_input(GatewaySample._gateway_new_location_request_string) - self._device.config.set_location(new_location) - print("The Gateway's location was set to {new_location}".format(new_location=new_location)) - - def _configure_volume(self): - print("Configuring the Gateway's volume") - self._volume_name = GatewaySample._get_input(GatewaySample._gateway_volume_name_request_string) - self._device.volumes.delete_all() - self._device.drive.format_all() - self._device.volumes.add(self._volume_name) - self._device.afp.disable() - self._device.ftp.disable() - self._device.rsync.disable() - print("The volume {volume_name} was added successfully".format(volume_name=self._volume_name)) - - def _set_dns_server(self): - print("Setting the Gateway's DNS Server to 8.8.8.8") - ipaddr = self._device.api.get('/status/network/ports/0/ip') - self._device.network.set_static_ipaddr(ipaddr.address, ipaddr.netmask, ipaddr.gateway, '8.8.8.8') - print("DNS Server was set to 8.8.8.8") - - def _connect_to_portal(self): - print("Connecting the Gateway to the Portal") - address = GatewaySample._get_input(GatewaySample._gateway_portal_address_request_string) - username = GatewaySample._get_input(GatewaySample._gateway_username_request_string) - password = GatewaySample._get_password(username) - self._device.services.connect(address, username, password) - print("Successfully connected to Portal at {address}".format(address=address)) - - def _start_cloud_caching(self): - print("Starting Cloud Sync") - self._device.cache.enable() - self._device.sync.unsuspend() - self._device.sync.refresh() - print("Cloud Sync was configured successfully") - - def _disable_first_time_wizard(self): - print("Disabling the First Time Wizard") - self._device.api.put('/config/gui/openFirstTimeWizard', False) - - def _create_share(self): - print("Creating a new share") - share_dir_base_path = 'public' - directory_name = self._get_input(GatewaySample._share_directory_name_request_string) - directory_path = '/'.join([share_dir_base_path, directory_name]) - self._device.files.mkdir(directory_path, recurse=True) - self._device.shares.add( - directory_name, - '/'.join([self._volume_name, directory_path]), - acl=[ - edge_types.ShareAccessControlEntry( - principal_type=edge_enum.PrincipalType.LG, - name='Everyone', - perm=edge_enum.FileAccessMode.RW - ) - ], - access=edge_enum.Acl.OnlyAuthenticatedUsers - ) - self._device.shares.add_acl( - directory_name, - [ - edge_types.ShareAccessControlEntry( - principal_type=edge_enum.PrincipalType.LG, - name='Administrators', - perm=edge_enum.FileAccessMode.RO - ) - ] - ) - self._device.shares.remove_acl( - directory_name, - acl=[ - edge_types.RemoveShareAccessControlEntry( - principal_type=edge_enum.PrincipalType.LG, - name='Everyone' - ) - ] - ) - print("New share was created successfully") - - -if __name__ == "__main__": - config.Logging.get().setLevel(logging.DEBUG) - try: - GatewaySample().run() - except CTERAException as error: - print(error) diff --git a/samples/portal_simple_sample.py b/samples/portal_simple_sample.py deleted file mode 100644 index 3c3c8f52..00000000 --- a/samples/portal_simple_sample.py +++ /dev/null @@ -1,84 +0,0 @@ -from cterasdk import CTERAException, GlobalAdmin, core_enum, edge_enum, edge_types -import cterasdk.settings - -from sample_base import CTERASDKSampleBase - - -class PortalSimpleSample(CTERASDKSampleBase): - _portal_address_request_string = "the Portal Address" - _portal_username_request_string = "the user name" - _portal_device_name_request_string = "the device name" - _share_directory_name_request_string = "a new for the new shared directory" - - def __init__(self): - self._global_admin = None - - def run(self): - self._connect_to_portal() - self._handle_users() - - print("Printing all devices") - self._print_devices() - - print("Printing only vGateways") - self._print_devices(deviceTypes=['vGateway'], include=['deviceReportedStatus']) - - self._remote_device_operations() - self._print_system_logs() - - self._global_admin.logout() - - def _connect_to_portal(self): - gateway_address = CTERASDKSampleBase._get_input(PortalSimpleSample._portal_address_request_string) - cterasdk.settings.sessions.management.ssl = False - self._global_admin = GlobalAdmin(gateway_address) - username = CTERASDKSampleBase._get_input(PortalSimpleSample._portal_username_request_string) - password = CTERASDKSampleBase._get_password(username) - self._global_admin.login(username, password) - - def _handle_users(self): - print("Creating a new user") - self._global_admin.users.add('alice', 'walice@acme.com', 'Alice', 'Wonderland', 'password1!', core_enum.Role.ReadWriteAdmin) - print("User was created succussfully") - - print("Deleting previously created user") - self._global_admin.users.delete('alice') - print("User was deleted successfully") - - def _print_devices(self, deviceTypes=None, include=None): - for device in self._global_admin.devices.filers(include=include, deviceTypes=deviceTypes): - print(device) - - def _remote_device_operations(self): - print("Creating a new share") - device_name = CTERASDKSampleBase._get_input(PortalSimpleSample._portal_device_name_request_string) - device = self._global_admin.devices.device(device_name) - share_dir_base_path = 'public' - directory_name = self._get_input(PortalSimpleSample._share_directory_name_request_string) - directory_path = '/'.join([share_dir_base_path, directory_name]) - device.shares.add( - directory_name, - '/'.join(['main', directory_path]), - acl=[ - edge_types.ShareAccessControlEntry(type='LG', name='Everyone', perm='RW'), - edge_types.ShareAccessControlEntry(type='LG', name='Administrators', perm='RO') - ], - access=edge_enum.Acl.OnlyAuthenticatedUsers - ) - print("New Share was created successfully") - print("Deleting previosly created share") - device.shares.delete(directory_name) - print("Share was deleted successfully") - - def _print_system_logs(self): - print("Printing Cloud Sync logs") - for log in self._global_admin.logs.logs(topic=core_enum.LogTopic.CloudSync): - print(log.msg) - - -if __name__ == "__main__": - try: - PortalSimpleSample().run() - except CTERAException as error: - - print(error) diff --git a/samples/remove_devices.py b/samples/remove_devices.py deleted file mode 100644 index 8f2dde26..00000000 --- a/samples/remove_devices.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging -import csv - -from cterasdk import CTERAException, GlobalAdmin -import cterasdk.settings - -from sample_base import CTERASDKSampleBase - - -class PortalRemoveDevicesSample(CTERASDKSampleBase): - _portal_address_request_string = "the Portal Address" - _portal_username_request_string = "the user name" - - def __init__(self): - self._global_admin = None - - def run(self): - cterasdk.settings.sessions.management.ssl = False - self._connect_to_portal() - - self._global_admin.portals.browse('acme') - - with open('devices.csv', newline='\n') as csvfile: - spamreader = csv.reader(csvfile, delimiter=',', quotechar='"') - for row in spamreader: - device_name = row[0] - logging.getLogger().info('Deleting device. %s', {'name' : device_name}) - try: - self._global_admin.execute('/devices/' + device_name, 'delete', 'deviceAndFolders') - logging.getLogger().info('Device deleted. %s', {'name' : device_name}) - except CTERAException: - logging.getLogger().error('Failed deleting device. %s', {'name' : device_name}) - - self._global_admin.logout() - - def _connect_to_portal(self): - gateway_address = PortalRemoveDevicesSample._get_input(PortalRemoveDevicesSample._portal_address_request_string) - self._global_admin = GlobalAdmin(gateway_address) - username = PortalRemoveDevicesSample._get_input(PortalRemoveDevicesSample._portal_username_request_string) - password = PortalRemoveDevicesSample._get_password(username) - self._global_admin.login(username, password) - - -if __name__ == "__main__": - PortalRemoveDevicesSample().run() diff --git a/samples/sample_base.py b/samples/sample_base.py deleted file mode 100644 index 3b1fd7cc..00000000 --- a/samples/sample_base.py +++ /dev/null @@ -1,19 +0,0 @@ -import getpass - -class CTERASDKSampleBase: - _password_request_string = "the password for {username}" - - @staticmethod - def _get_password(username): - return CTERASDKSampleBase._get_input(CTERASDKSampleBase._password_request_string.format(username=username), echo=False) - - @staticmethod - def _get_input(prompt, echo=True): - prompt = "Please enter {prompt}: ".format(prompt=prompt) - response = None - while not response: - if echo: - response = input(prompt) - else: - response = getpass.getpass(prompt) - return response diff --git a/samples/services_portal_create_directories_sample.py b/samples/services_portal_create_directories_sample.py deleted file mode 100755 index 70dd7e17..00000000 --- a/samples/services_portal_create_directories_sample.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python3 -from cterasdk import ServicesPortal, CTERAException -import cterasdk.settings - -from sample_base import CTERASDKSampleBase - - -class ServicesPortalCreateDirectoriesSample(CTERASDKSampleBase): - _portal_address_request_string = "the Portal Address" - _portal_username_request_string = "the user name" - _directory_name_request_string = "the directory name" - - def __init__(self): - self._services_portal = None - - def run(self): - cterasdk.settings.sessions.management.ssl = False - self._connect_to_portal() - - self._create_directories() - - self._services_portal.logout() - - def _connect_to_portal(self): - gateway_address = CTERASDKSampleBase._get_input(ServicesPortalCreateDirectoriesSample._portal_address_request_string) - self._services_portal = ServicesPortal(gateway_address) - username = CTERASDKSampleBase._get_input(ServicesPortalCreateDirectoriesSample._portal_username_request_string) - password = CTERASDKSampleBase._get_password(username) - self._services_portal.login(username, password) - - def _create_directories(self): - directory_name = CTERASDKSampleBase._get_input(ServicesPortalCreateDirectoriesSample._directory_name_request_string) - self._services_portal.files.mkdir(directory_name, recurse=True) - - -if __name__ == "__main__": - try: - ServicesPortalCreateDirectoriesSample().run() - except CTERAException as error: - print(error) diff --git a/tests/ut/edge/test_config.py b/tests/ut/edge/test_config.py index c74a7271..98eda7f6 100644 --- a/tests/ut/edge/test_config.py +++ b/tests/ut/edge/test_config.py @@ -15,7 +15,7 @@ def setUp(self): self._location = '205 E. 42nd St. New York, NY. 10017' self._filename = 'file.xml' self._target_directory = '/path/to/folder' - self._default_download_directory = cterasdk.settings.downloads.location + self._default_download_directory = cterasdk.settings.io.downloads self._current_datetime = datetime.datetime.now() def test_get_hostname(self): diff --git a/tests/ut/edge/test_files_browser.py b/tests/ut/edge/test_files_browser.py index edbf10d7..bf27f8a0 100644 --- a/tests/ut/edge/test_files_browser.py +++ b/tests/ut/edge/test_files_browser.py @@ -16,7 +16,7 @@ def setUp(self): self._fullpath = '/' + self._path self._target = 'target/folder' self._target_fullpath = f'/{self._target}/{self._filename}' - self._default_download_dir = cterasdk.settings.downloads.location + self._default_download_dir = cterasdk.settings.io.downloads def test_download_as_zip_success(self): pass # self._files.download_as_zip() diff --git a/tests/ut/edge/test_support.py b/tests/ut/edge/test_support.py index bb0325e0..1e1b8275 100644 --- a/tests/ut/edge/test_support.py +++ b/tests/ut/edge/test_support.py @@ -31,9 +31,9 @@ def test_get_support_report(self): current_datetime = datetime.datetime.now() handle_response = 'Stream' self._init_filer(handle_response=handle_response) - mock_save_file = self.patch_call("cterasdk.lib.filesystem.FileSystem.save") + mock_save_file = self.patch_call("cterasdk.lib.storage.synfs.write") with freeze_time(current_datetime): support.Support(self._filer).get_support_report() self._filer.api.handle.assert_called_once_with('/supportreport') filename = 'Support-' + current_datetime.strftime('_%Y-%m-%dT%H_%M_%S') + '.zip' - mock_save_file.assert_called_once_with(cterasdk.settings.downloads.location, filename, handle_response) + mock_save_file.assert_called_once_with(cterasdk.settings.io.downloads, filename, handle_response) From 59df2857c6c6cfcb6af294a6ba4fa3df9015b0e1 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 18:57:04 -0400 Subject: [PATCH 04/12] resolve ut errors --- cterasdk/settings.py | 2 +- tests/ut/aio/base.py | 4 ++++ tests/ut/base.py | 4 ++++ tests/ut/edge/test_ssh.py | 2 +- tests/ut/edge/test_support.py | 3 ++- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cterasdk/settings.py b/cterasdk/settings.py index 2c21aaf3..7cb7f34e 100644 --- a/cterasdk/settings.py +++ b/cterasdk/settings.py @@ -22,4 +22,4 @@ settings = fromjsonstr(json.dumps(yaml.safe_load(f))) -core, edge, io, audit = settings.core, settings.edge, settings.io, settings.audit +core, edge, drive, io, audit = settings.core, settings.edge, settings.drive, settings.io, settings.audit diff --git a/tests/ut/aio/base.py b/tests/ut/aio/base.py index 13548b4b..f23d4d03 100644 --- a/tests/ut/aio/base.py +++ b/tests/ut/aio/base.py @@ -5,6 +5,10 @@ class BaseAsyncTest(unittest.IsolatedAsyncioTestCase): """Base Async Test""" + def setUp(self): + super().setUp() + self.patch_call('asyncio.get_running_loop') + def patch_call(self, module_path, **patch_kwargs): return utilities.patch_call(self, module_path, **patch_kwargs) diff --git a/tests/ut/base.py b/tests/ut/base.py index b004871c..79896f48 100644 --- a/tests/ut/base.py +++ b/tests/ut/base.py @@ -4,6 +4,10 @@ class BaseTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.patch_call('asyncio.get_running_loop') + def patch_call(self, module_path, **patch_kwargs): return utilities.patch_call(self, module_path, **patch_kwargs) diff --git a/tests/ut/edge/test_ssh.py b/tests/ut/edge/test_ssh.py index e0f3dffe..1e820829 100644 --- a/tests/ut/edge/test_ssh.py +++ b/tests/ut/edge/test_ssh.py @@ -37,7 +37,7 @@ def test_enable_no_public_key(self): def test_enable_public_key_from_file(self): response = 'PublicKey' self._init_filer() - self.patch_call('cterasdk.edge.ssh.FileSystem.properties') + self.patch_call('cterasdk.edge.ssh.commonfs.properties') with mock.patch("builtins.open", mock.mock_open(read_data=response)): ssh.SSH(self._filer).enable(public_key_file=self._public_key_file) self._filer.api.execute.assert_called_once_with('/config/device', 'startSSHD', mock.ANY) diff --git a/tests/ut/edge/test_support.py b/tests/ut/edge/test_support.py index 1e1b8275..e7ff9d08 100644 --- a/tests/ut/edge/test_support.py +++ b/tests/ut/edge/test_support.py @@ -1,5 +1,6 @@ import datetime +from pathlib import Path from freezegun import freeze_time from cterasdk import exceptions @@ -36,4 +37,4 @@ def test_get_support_report(self): support.Support(self._filer).get_support_report() self._filer.api.handle.assert_called_once_with('/supportreport') filename = 'Support-' + current_datetime.strftime('_%Y-%m-%dT%H_%M_%S') + '.zip' - mock_save_file.assert_called_once_with(cterasdk.settings.io.downloads, filename, handle_response) + mock_save_file.assert_called_once_with(Path(cterasdk.settings.io.downloads).expanduser(), filename, handle_response) From dff5afcb90b28758d7d64a513f60cd5cfceea0ec Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 18:59:27 -0400 Subject: [PATCH 05/12] resolve flake errors --- cterasdk/__init__.py | 2 +- cterasdk/clients/base.py | 2 +- cterasdk/objects/asynchronous/edge.py | 2 +- cterasdk/objects/services.py | 1 + cterasdk/settings.py | 1 - 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cterasdk/__init__.py b/cterasdk/__init__.py index 474b5ae8..2ed45f73 100644 --- a/cterasdk/__init__.py +++ b/cterasdk/__init__.py @@ -11,5 +11,5 @@ from .core import enum as core_enum # noqa: E402, F401 from .common import types as common_types # noqa: E402, F401 from .common import enum as common_enum # noqa: E402, F401 -from .objects import GlobalAdmin, ServicesPortal, Edge, Drive, AsyncGlobalAdmin, AsyncServicesPortal, AsyncEdge # noqa: E402, F401 +from .objects import GlobalAdmin, ServicesPortal, Edge, Drive, AsyncGlobalAdmin, AsyncServicesPortal, AsyncEdge # noqa: E402, F401 from . import direct as ctera_direct # noqa: E402, F401 diff --git a/cterasdk/clients/base.py b/cterasdk/clients/base.py index 238d6c9c..86ed9c29 100644 --- a/cterasdk/clients/base.py +++ b/cterasdk/clients/base.py @@ -69,7 +69,7 @@ def __init__(self, builder=None, session=None, settings=None, authenticator=None default_settings = ClientSessionSettings() if settings: default_settings.update(**settings.kwargs) - + self._session = session if session else async_requests.Session(**default_settings, **TraceSettings()) def clone(self, definition, builder=None, authenticator=None): diff --git a/cterasdk/objects/asynchronous/edge.py b/cterasdk/objects/asynchronous/edge.py index 1c5ba050..7f660fdf 100644 --- a/cterasdk/objects/asynchronous/edge.py +++ b/cterasdk/objects/asynchronous/edge.py @@ -74,4 +74,4 @@ def _login_object(self): return login.Login(self) def _authenticator(self, url): - return authenticators.edge(self.session(), url) \ No newline at end of file + return authenticators.edge(self.session(), url) diff --git a/cterasdk/objects/services.py b/cterasdk/objects/services.py index 05fcb871..eb51aed8 100644 --- a/cterasdk/objects/services.py +++ b/cterasdk/objects/services.py @@ -102,6 +102,7 @@ async def logout(self): async def __aexit__(self, exc_type, exc, tb): await self.default.close() + class Management(CTERA): def __enter__(self): diff --git a/cterasdk/settings.py b/cterasdk/settings.py index 7cb7f34e..62e740bc 100644 --- a/cterasdk/settings.py +++ b/cterasdk/settings.py @@ -9,7 +9,6 @@ logger = logging.getLogger('cterasdk') - settings = Path(__file__).parent.absolute().joinpath('settings.yml') try: commonfs.properties(settings) From f73ea00e8ac8e5ba5951f1cdcda548799a703e61 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:00:36 -0400 Subject: [PATCH 06/12] unused module --- cterasdk/core/files/browser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index 8be6ff53..a564791d 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -1,6 +1,5 @@ import logging -import cterasdk.settings from ...cio.core import CorePath from ...exceptions import CTERAException from ...lib.storage import synfs, commonfs From 8cbb27df715d4bec77f6570ca81b03adb1f2533e Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:08:18 -0400 Subject: [PATCH 07/12] lint errors --- cterasdk/clients/settings.py | 10 +++++----- cterasdk/objects/services.py | 2 +- cterasdk/settings.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cterasdk/clients/settings.py b/cterasdk/clients/settings.py index 243c246d..d5dc935f 100644 --- a/cterasdk/clients/settings.py +++ b/cterasdk/clients/settings.py @@ -1,7 +1,7 @@ import copy -import cterasdk.settings from collections.abc import MutableMapping from aiohttp import TCPConnector, CookieJar, ClientTimeout +import cterasdk.settings from .tracers import requests, session, postman @@ -25,7 +25,7 @@ def __init__(self, *args, **kwargs): } self._mapping.update(dict(*args, **kwargs)) - def update(self, **kwargs): + def update(self, **kwargs): # pylint: disable=arguments-differ for k, v in self._mapping.items(): attributes = kwargs.get(k, None) self._mapping[k] = attributes @@ -67,13 +67,13 @@ def __init__(self): def __getitem__(self, key): return self._mapping.get(key, None) - def __setitem__(self, key, value): + def __setitem__(self, key, value): # pylint: disable=useless-parent-delegation return super().__setitem__(key, value) - def __delitem__(self, key): + def __delitem__(self, key): # pylint: disable=useless-parent-delegation return super().__delitem__(key) - def __len__(self): + def __len__(self): # pylint: disable=useless-parent-delegation return super().__len__() def __iter__(self): diff --git a/cterasdk/objects/services.py b/cterasdk/objects/services.py index eb51aed8..179ca583 100644 --- a/cterasdk/objects/services.py +++ b/cterasdk/objects/services.py @@ -74,7 +74,7 @@ def whoami(self): return self._ctera_session.whoami() -class AsyncManagement(CTERA): +class AsyncManagement(CTERA): # pylint: disable=abstract-method async def __aenter__(self): return self diff --git a/cterasdk/settings.py b/cterasdk/settings.py index 62e740bc..6289e2e5 100644 --- a/cterasdk/settings.py +++ b/cterasdk/settings.py @@ -1,7 +1,7 @@ -import yaml import json import logging from pathlib import Path +import yaml from .convert import fromjsonstr from .lib.storage import commonfs From 6fbfe3bf01c0e81340963cf886509b98363fbc38 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:12:35 -0400 Subject: [PATCH 08/12] resolve circular import --- cterasdk/settings.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cterasdk/settings.py b/cterasdk/settings.py index 6289e2e5..435434f6 100644 --- a/cterasdk/settings.py +++ b/cterasdk/settings.py @@ -3,7 +3,6 @@ from pathlib import Path import yaml from .convert import fromjsonstr -from .lib.storage import commonfs logger = logging.getLogger('cterasdk') @@ -11,14 +10,9 @@ settings = Path(__file__).parent.absolute().joinpath('settings.yml') try: - commonfs.properties(settings) + with open(settings, 'r', encoding='utf-8') as f: + settings = fromjsonstr(json.dumps(yaml.safe_load(f))) + core, edge, drive, io, audit = settings.core, settings.edge, settings.drive, settings.io, settings.audit except FileNotFoundError: logger.fatal("Configuration file 'settings.yml' not found. Please check your installation and try again.") raise - - -with open(settings, 'r', encoding='utf-8') as f: - settings = fromjsonstr(json.dumps(yaml.safe_load(f))) - - -core, edge, drive, io, audit = settings.core, settings.edge, settings.drive, settings.io, settings.audit From 51b983135ed31f1abc7d9a6f6e866b07a04413ae Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:23:38 -0400 Subject: [PATCH 09/12] resolve doc errors --- docs/source/UserGuides/Edge/Files.rst | 57 +++++++++++++++++-- .../api/cterasdk.clients.tracers.logger.rst | 6 +- docs/source/api/cterasdk.clients.tracers.rst | 2 +- docs/source/api/cterasdk.lib.filesystem.rst | 7 --- docs/source/api/cterasdk.lib.rst | 2 +- docs/source/api/cterasdk.lib.storage.rst | 16 ++++++ 6 files changed, 72 insertions(+), 18 deletions(-) delete mode 100644 docs/source/api/cterasdk.lib.filesystem.rst create mode 100644 docs/source/api/cterasdk.lib.storage.rst diff --git a/docs/source/UserGuides/Edge/Files.rst b/docs/source/UserGuides/Edge/Files.rst index f68367c2..0da4ca73 100644 --- a/docs/source/UserGuides/Edge/Files.rst +++ b/docs/source/UserGuides/Edge/Files.rst @@ -2,9 +2,11 @@ File Browser ============ +Synchronous API +=============== List -==== +---- .. automethod:: cterasdk.edge.files.browser.FileBrowser.listdir :noindex: @@ -15,7 +17,7 @@ List print(item.name, item.fullpath) Download -======== +-------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.download :noindex: @@ -32,7 +34,7 @@ Download edge.files.download_many('network-share/docs', ['Sample.docx', 'Summary.xlsx']) Create Directory -================ +---------------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.mkdir :noindex: @@ -51,7 +53,7 @@ Create Directory Copy -==== +---- .. automethod:: cterasdk.edge.files.browser.FileBrowser.copy :noindex: @@ -69,7 +71,7 @@ Copy Move -==== +---- .. automethod:: cterasdk.edge.files.browser.FileBrowser.move :noindex: @@ -86,7 +88,7 @@ Move edge.files.move('cloud/users/Bruce Wayne/My Files/Summary.xlsx', destination='cloud/users/Bruce Wayne/Spreadsheets', overwrite=True) Delete -====== +------ .. automethod:: cterasdk.edge.files.browser.FileBrowser.delete :noindex: @@ -94,3 +96,46 @@ Delete .. code:: python edge.files.delete('cloud/users/Service Account/My Files/Documents') + + +Asynchronous API +================ + +Asynchronous API +================ + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.handle + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.handle_many + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.download + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.download_many + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.listdir + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.copy + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.move + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.upload + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.upload_file + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.mkdir + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.makedirs + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.delete + :noindex: \ No newline at end of file diff --git a/docs/source/api/cterasdk.clients.tracers.logger.rst b/docs/source/api/cterasdk.clients.tracers.logger.rst index 5f7429aa..b05f101d 100644 --- a/docs/source/api/cterasdk.clients.tracers.logger.rst +++ b/docs/source/api/cterasdk.clients.tracers.logger.rst @@ -1,7 +1,7 @@ -cterasdk.clients.tracers.logger module -====================================== +cterasdk.clients.tracers.requests module +======================================== -.. automodule:: cterasdk.clients.tracers.logger +.. automodule:: cterasdk.clients.tracers.requests :members: :undoc-members: :show-inheritance: diff --git a/docs/source/api/cterasdk.clients.tracers.rst b/docs/source/api/cterasdk.clients.tracers.rst index 6e3242b1..3864e1ca 100644 --- a/docs/source/api/cterasdk.clients.tracers.rst +++ b/docs/source/api/cterasdk.clients.tracers.rst @@ -11,6 +11,6 @@ Submodules .. toctree:: - cterasdk.clients.tracers.logger cterasdk.clients.tracers.postman + cterasdk.clients.tracers.requests cterasdk.clients.tracers.session diff --git a/docs/source/api/cterasdk.lib.filesystem.rst b/docs/source/api/cterasdk.lib.filesystem.rst deleted file mode 100644 index 50ce887a..00000000 --- a/docs/source/api/cterasdk.lib.filesystem.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.lib.filesystem module -============================== - -.. automodule:: cterasdk.lib.filesystem - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.lib.rst b/docs/source/api/cterasdk.lib.rst index 6ba39bc7..b8a97433 100644 --- a/docs/source/api/cterasdk.lib.rst +++ b/docs/source/api/cterasdk.lib.rst @@ -13,10 +13,10 @@ Submodules cterasdk.lib.cmd cterasdk.lib.consent - cterasdk.lib.filesystem cterasdk.lib.iterator cterasdk.lib.platform cterasdk.lib.registry + cterasdk.lib.storage cterasdk.lib.tempfile cterasdk.lib.tracker cterasdk.lib.version diff --git a/docs/source/api/cterasdk.lib.storage.rst b/docs/source/api/cterasdk.lib.storage.rst new file mode 100644 index 00000000..12fef649 --- /dev/null +++ b/docs/source/api/cterasdk.lib.storage.rst @@ -0,0 +1,16 @@ +cterasdk.lib.storage package +============================ + +.. automodule:: cterasdk.lib.storage + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + cterasdk.lib.storage.asynfs + cterasdk.lib.storage.commonfs + cterasdk.lib.storage.synfs From 07973c5f7a10dba2117061a26af11df155de2884 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:27:23 -0400 Subject: [PATCH 10/12] continue to resolve doc issues --- docs/source/UserGuides/Edge/Files.rst | 10 +++++----- ...ogger.rst => cterasdk.clients.tracers.requests.rst} | 0 docs/source/api/cterasdk.lib.storage.asynfs.rst | 7 +++++++ docs/source/api/cterasdk.lib.storage.commonfs.rst | 7 +++++++ docs/source/api/cterasdk.lib.storage.synfs.rst | 7 +++++++ 5 files changed, 26 insertions(+), 5 deletions(-) rename docs/source/api/{cterasdk.clients.tracers.logger.rst => cterasdk.clients.tracers.requests.rst} (100%) create mode 100644 docs/source/api/cterasdk.lib.storage.asynfs.rst create mode 100644 docs/source/api/cterasdk.lib.storage.commonfs.rst create mode 100644 docs/source/api/cterasdk.lib.storage.synfs.rst diff --git a/docs/source/UserGuides/Edge/Files.rst b/docs/source/UserGuides/Edge/Files.rst index 0da4ca73..543bde41 100644 --- a/docs/source/UserGuides/Edge/Files.rst +++ b/docs/source/UserGuides/Edge/Files.rst @@ -125,17 +125,17 @@ Asynchronous API .. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.move :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.upload +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.upload_file +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload_file :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.mkdir +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.mkdir :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.makedirs +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.makedirs :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.CloudDrive.delete +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.delete :noindex: \ No newline at end of file diff --git a/docs/source/api/cterasdk.clients.tracers.logger.rst b/docs/source/api/cterasdk.clients.tracers.requests.rst similarity index 100% rename from docs/source/api/cterasdk.clients.tracers.logger.rst rename to docs/source/api/cterasdk.clients.tracers.requests.rst diff --git a/docs/source/api/cterasdk.lib.storage.asynfs.rst b/docs/source/api/cterasdk.lib.storage.asynfs.rst new file mode 100644 index 00000000..466a5c29 --- /dev/null +++ b/docs/source/api/cterasdk.lib.storage.asynfs.rst @@ -0,0 +1,7 @@ +cterasdk.lib.storage.asynfs module +================================== + +.. automodule:: cterasdk.lib.storage.asynfs + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.lib.storage.commonfs.rst b/docs/source/api/cterasdk.lib.storage.commonfs.rst new file mode 100644 index 00000000..b55eb09a --- /dev/null +++ b/docs/source/api/cterasdk.lib.storage.commonfs.rst @@ -0,0 +1,7 @@ +cterasdk.lib.storage.commonfs module +==================================== + +.. automodule:: cterasdk.lib.storage.commonfs + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.lib.storage.synfs.rst b/docs/source/api/cterasdk.lib.storage.synfs.rst new file mode 100644 index 00000000..efecfb0a --- /dev/null +++ b/docs/source/api/cterasdk.lib.storage.synfs.rst @@ -0,0 +1,7 @@ +cterasdk.lib.storage.synfs module +================================= + +.. automodule:: cterasdk.lib.storage.synfs + :members: + :undoc-members: + :show-inheritance: From 1590f9f8bf5eb3c5a1be1e7b54f8c720e4f91944 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 19:29:08 -0400 Subject: [PATCH 11/12] resolve doc move --- docs/source/UserGuides/Portal/Files.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/UserGuides/Portal/Files.rst b/docs/source/UserGuides/Portal/Files.rst index 03975e81..15c5efd9 100644 --- a/docs/source/UserGuides/Portal/Files.rst +++ b/docs/source/UserGuides/Portal/Files.rst @@ -457,7 +457,7 @@ Asynchronous API .. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.copy :noindex: -.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.move +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.move :noindex: .. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.permalink From 528b3cfdf39acae95ef7ecd878c318eb78f82142 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Sun, 18 May 2025 20:39:05 -0400 Subject: [PATCH 12/12] resolve lint and ut --- docs/source/UserGuides/Edge/Files.rst | 2 +- tests/ut/edge/test_support.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/UserGuides/Edge/Files.rst b/docs/source/UserGuides/Edge/Files.rst index 543bde41..03a8a1d3 100644 --- a/docs/source/UserGuides/Edge/Files.rst +++ b/docs/source/UserGuides/Edge/Files.rst @@ -138,4 +138,4 @@ Asynchronous API :noindex: .. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.delete - :noindex: \ No newline at end of file + :noindex: diff --git a/tests/ut/edge/test_support.py b/tests/ut/edge/test_support.py index e7ff9d08..9f0d4d0f 100644 --- a/tests/ut/edge/test_support.py +++ b/tests/ut/edge/test_support.py @@ -29,6 +29,7 @@ def test_set_debug_level_input_error(self): self.assertEqual('Invalid debug level', error.exception.message) def test_get_support_report(self): + cterasdk.settings.io.downloads = '~' current_datetime = datetime.datetime.now() handle_response = 'Stream' self._init_filer(handle_response=handle_response) @@ -37,4 +38,4 @@ def test_get_support_report(self): support.Support(self._filer).get_support_report() self._filer.api.handle.assert_called_once_with('/supportreport') filename = 'Support-' + current_datetime.strftime('_%Y-%m-%dT%H_%M_%S') + '.zip' - mock_save_file.assert_called_once_with(Path(cterasdk.settings.io.downloads).expanduser(), filename, handle_response) + mock_save_file.assert_called_once_with(Path('~').expanduser(), filename, handle_response)