From dd189487556c97619954b346c97ae7047e3df470 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Wed, 28 May 2025 17:07:57 -0400 Subject: [PATCH 1/6] make Object a mutable mapping and introduce support for propfind requests --- cterasdk/asynchronous/core/files/browser.py | 7 +++---- cterasdk/clients/async_requests.py | 7 +++++++ cterasdk/clients/clients.py | 16 ++++++++++++++++ cterasdk/clients/common.py | 3 ++- cterasdk/common/object.py | 20 ++++++++++++++++---- cterasdk/convert/__init__.py | 2 +- cterasdk/convert/deserializers.py | 4 ++++ cterasdk/core/files/browser.py | 6 +++--- cterasdk/objects/asynchronous/core.py | 4 ++++ cterasdk/objects/asynchronous/edge.py | 4 ++++ cterasdk/objects/synchronous/core.py | 4 ++++ cterasdk/objects/synchronous/edge.py | 4 ++++ 12 files changed, 68 insertions(+), 13 deletions(-) diff --git a/cterasdk/asynchronous/core/files/browser.py b/cterasdk/asynchronous/core/files/browser.py index 405d4d40..637f103e 100644 --- a/cterasdk/asynchronous/core/files/browser.py +++ b/cterasdk/asynchronous/core/files/browser.py @@ -115,10 +115,9 @@ async def permalink(self, path): :param str path: Path. """ p = self.normalize(path) - contents = [e async for e in await io.listdir(self._core, - p.parent, 1, False, p.name, 1)] # pylint: disable=unnecessary-comprehension - if contents and contents[0].name == p.name: - return contents[0].permalink + async for e in await io.listdir(self._core, p.parent, 1, False, p.name, 1): + if e.name == p.name: + return e.permalink raise FileNotFoundError('File not found.', path) def normalize(self, entries): diff --git a/cterasdk/clients/async_requests.py b/cterasdk/clients/async_requests.py index c80e025c..6d3ac117 100644 --- a/cterasdk/clients/async_requests.py +++ b/cterasdk/clients/async_requests.py @@ -134,6 +134,13 @@ def __init__(self, url, **kwargs): super().__init__('DELETE', url, **kwargs) +class PropfindRequest(BaseRequest): + """PROPFIND""" + + def __init__(self, url, **kwargs): + super().__init__('PROPFIND', url, **kwargs) + + class MkcolRequest(BaseRequest): """MKCOL""" diff --git a/cterasdk/clients/clients.py b/cterasdk/clients/clients.py index 7d94324f..207b121f 100644 --- a/cterasdk/clients/clients.py +++ b/cterasdk/clients/clients.py @@ -59,6 +59,11 @@ class AsyncWebDAV(AsyncClient): async def download(self, path, **kwargs): return await super().get(path, **kwargs) + async def propfind(self, path): + request = async_requests.PropfindRequest(self._builder(path)) + response = await self.async_request(request) + return await response.dav() + async def mkcol(self, path): request = async_requests.MkcolRequest(self._builder(path)) response = await self.async_request(request) @@ -175,6 +180,9 @@ async def json(self): async def xml(self): return Deserializers.XML(await self._response.read()) + async def dav(self): + return Deserializers.DAV(await self._response.read()) + @async_requests.decorate_stream_error async def read(self, n=-1): return await self._response.content.read(n) @@ -251,6 +259,11 @@ class WebDAV(Client): def download(self, path, **kwargs): return super().handle(path, **kwargs) + def propfind(self, path): + request = async_requests.PropfindRequest(self._builder(path)) + response = self.request(request) + return response.dav() + def mkcol(self, path): request = async_requests.MkcolRequest(self._builder(path)) response = self.request(request) @@ -403,6 +416,9 @@ def json(self): # pylint: disable=invalid-overridden-method def xml(self): # pylint: disable=invalid-overridden-method return execute(super().xml) + def dav(self): # pylint: disable=invalid-overridden-method + return execute(super().dav) + @staticmethod def new(): async def new_response(response): diff --git a/cterasdk/clients/common.py b/cterasdk/clients/common.py index fe79173e..ca57295b 100644 --- a/cterasdk/clients/common.py +++ b/cterasdk/clients/common.py @@ -1,5 +1,5 @@ from . import async_requests -from ..convert import tojsonstr, toxmlstr, fromjsonstr, fromxmlstr +from ..convert import tojsonstr, toxmlstr, fromjsonstr, fromxmlstr, fromdavxmlstr class Serializers: @@ -11,6 +11,7 @@ class Serializers: class Deserializers: JSON = fromjsonstr XML = fromxmlstr + DAV = fromdavxmlstr class MultipartForm: diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index a7468f66..8276cf19 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -1,13 +1,25 @@ import re import json import logging +from collections.abc import MutableMapping -class Object: # pylint: disable=too-many-instance-attributes +class Object(MutableMapping): # pylint: disable=too-many-instance-attributes - @property - def kwargs(self): - return json.loads(str(self)) + def __getitem__(self, key): + return getattr(self, key, None) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __delitem__(self, key): + delattr(self, key) + + def __iter__(self): + return iter(self.__dict__) + + def __len__(self): + return len(self.__dict__) def __str__(self): return json.dumps(self, default=lambda o: o.__dict__, indent=5) diff --git a/cterasdk/convert/__init__.py b/cterasdk/convert/__init__.py index e827bee9..29c6a097 100644 --- a/cterasdk/convert/__init__.py +++ b/cterasdk/convert/__init__.py @@ -1,2 +1,2 @@ -from .deserializers import fromjsonstr, fromxmlstr # noqa: E402, F401 +from .deserializers import fromjsonstr, fromxmlstr, fromdavxmlstr # noqa: E402, F401 from .serializers import tojsonstr, toxmlstr # noqa: E402, F401 diff --git a/cterasdk/convert/deserializers.py b/cterasdk/convert/deserializers.py index b4639918..ac41b215 100644 --- a/cterasdk/convert/deserializers.py +++ b/cterasdk/convert/deserializers.py @@ -84,6 +84,10 @@ def fromjsonstr(fromstr): return root.value +def fromdavxmlstr(string): + return fromstring(string) + + def fromxmlstr(string): # pylint: disable=too-many-branches,too-many-statements if not string: diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index a564791d..6ed9e461 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -118,9 +118,9 @@ def permalink(self, path): :param str path: Path. """ p = self.normalize(path) - contents = [e for e in io.listdir(self._core, p.parent, 1, False, p.name, 1)] # pylint: disable=unnecessary-comprehension - if contents and contents[0].name == p.name: - return contents[0].permalink + for e in io.listdir(self._core, p.parent, 1, False, p.name, 1): + if e.name == p.name: + return e.permalink raise FileNotFoundError('File not found.', path) def normalize(self, entries): diff --git a/cterasdk/objects/asynchronous/core.py b/cterasdk/objects/asynchronous/core.py index 94a2f78c..f11a4bdd 100644 --- a/cterasdk/objects/asynchronous/core.py +++ b/cterasdk/objects/asynchronous/core.py @@ -39,6 +39,10 @@ def __init__(self, core): def upload(self): return self._upload.upload + @property + def propfind(self): + return self._webdav.propfind + @property def download(self): return self._webdav.get diff --git a/cterasdk/objects/asynchronous/edge.py b/cterasdk/objects/asynchronous/edge.py index 7f660fdf..dd4f36ce 100644 --- a/cterasdk/objects/asynchronous/edge.py +++ b/cterasdk/objects/asynchronous/edge.py @@ -32,6 +32,10 @@ def download_zip(self): def upload(self): return self._edge.default.form_data # pylint: disable=protected-access + @property + def propfind(self): + return self._webdav.propfind + @property def mkdir(self): return self._webdav.mkcol diff --git a/cterasdk/objects/synchronous/core.py b/cterasdk/objects/synchronous/core.py index 55dc53a3..9dedb080 100644 --- a/cterasdk/objects/synchronous/core.py +++ b/cterasdk/objects/synchronous/core.py @@ -32,6 +32,10 @@ def __init__(self, core): def upload(self): return self._upload.upload + @property + def propfind(self): + return self._webdav.propfind + @property def download(self): return self._webdav.download diff --git a/cterasdk/objects/synchronous/edge.py b/cterasdk/objects/synchronous/edge.py index 9d706ecc..50728c72 100644 --- a/cterasdk/objects/synchronous/edge.py +++ b/cterasdk/objects/synchronous/edge.py @@ -46,6 +46,10 @@ def download_zip(self): def upload(self): return self._edge.default.form_data # pylint: disable=protected-access + @property + def propfind(self): + return self._webdav.propfind + @property def mkdir(self): return self._webdav.mkcol From cddb128c264c9f17540a3be3836518c2e19838e6 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Wed, 28 May 2025 22:11:08 -0400 Subject: [PATCH 2/6] update object to pass ut for mutable mapping --- cterasdk/clients/base.py | 2 +- cterasdk/common/object.py | 6 +++++- cterasdk/edge/ctera_migrate.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cterasdk/clients/base.py b/cterasdk/clients/base.py index 94e07aa4..1d2ae989 100644 --- a/cterasdk/clients/base.py +++ b/cterasdk/clients/base.py @@ -68,7 +68,7 @@ def __init__(self, builder=None, session=None, settings=None, authenticator=None default_settings = ClientSessionSettings() if settings: - default_settings.update(**settings.kwargs) + default_settings.update(**settings) self._session = session if session else async_requests.Session(default_settings, TraceSettings()) diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index 8276cf19..7cc65552 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -6,8 +6,12 @@ class Object(MutableMapping): # pylint: disable=too-many-instance-attributes + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + def __getitem__(self, key): - return getattr(self, key, None) + return getattr(self, key) def __setitem__(self, key, value): setattr(self, key, value) diff --git a/cterasdk/edge/ctera_migrate.py b/cterasdk/edge/ctera_migrate.py index 03802c32..71023b0f 100644 --- a/cterasdk/edge/ctera_migrate.py +++ b/cterasdk/edge/ctera_migrate.py @@ -41,7 +41,7 @@ def list_tasks(self, deleted=False): :rtype: list(cterasdk.common.object.Object) """ tasks = self._edge.migrate.get('/tasks/list', params={'deleted': int(deleted)}).tasks # pylint: disable=W0212 - return [Task.from_server_object(task) for task in tasks.__dict__.values()] if tasks else [] + return [Task.from_server_object(task) for task in tasks.values()] if tasks else [] def delete(self, tasks): """ From 03521f3ed58e30a4106f4bd60189fac1b7223da6 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Wed, 28 May 2025 22:38:43 -0400 Subject: [PATCH 3/6] add super init to pass lint --- cterasdk/asynchronous/core/types.py | 1 + cterasdk/audit/postman.py | 7 +++++++ cterasdk/cio/core.py | 4 ++++ cterasdk/clients/errors.py | 1 + cterasdk/common/object.py | 1 + cterasdk/common/types.py | 7 +++++++ cterasdk/common/utils.py | 1 + cterasdk/core/query.py | 3 +++ cterasdk/core/types.py | 4 ++++ cterasdk/direct/types.py | 2 ++ cterasdk/edge/ctera_migrate.py | 1 + cterasdk/edge/query.py | 1 + cterasdk/edge/types.py | 1 + cterasdk/lib/session/base.py | 2 ++ cterasdk/lib/session/edge.py | 1 + 15 files changed, 37 insertions(+) diff --git a/cterasdk/asynchronous/core/types.py b/cterasdk/asynchronous/core/types.py index fb74fa5c..7a714421 100644 --- a/cterasdk/asynchronous/core/types.py +++ b/cterasdk/asynchronous/core/types.py @@ -21,6 +21,7 @@ def __init__( # pylint: disable=redefined-builtin, too-many-arguments self, type, guid, deleted, name, folder_id=None, modified=None, file_timestamp=None, size=None, id=None, acl=None, gvsn=None, parent_guid=None, portal_modified_date=None, virtual_portal_id=None): + super().__init__() self.type = type self.guid = guid self.folder_id = folder_id diff --git a/cterasdk/audit/postman.py b/cterasdk/audit/postman.py index 779dc9ad..0675eded 100644 --- a/cterasdk/audit/postman.py +++ b/cterasdk/audit/postman.py @@ -15,6 +15,7 @@ class Collection(Object): __instance = None def __init__(self): + super().__init__() self.info = Info() self.item = [] Collection.__instance = self @@ -36,6 +37,7 @@ def serialize(self): class Info(Object): def __init__(self): + super().__init__() self.name = f'{str(uuid.uuid4())}' self.schema = "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -43,6 +45,7 @@ def __init__(self): class Command(Object): def __init__(self, name, request): + super().__init__() self.name = name self.request = request self.response = [] @@ -51,6 +54,7 @@ def __init__(self, name, request): class Request(Object): def __init__(self, method, url): + super().__init__() self.method = method self.header = None self.url = url @@ -65,6 +69,7 @@ def request_body(self, data): class Header(Object): def __init__(self, key, value): + super().__init__() self.key = key self.value = value self.type = 'text' @@ -159,6 +164,7 @@ class Body(Object): """Request Body""" def __init__(self, mode): + super().__init__() self.mode = mode @@ -217,6 +223,7 @@ def json(body): class URL(Object): def __init__(self, raw, protocol, host, port, path, query): + super().__init__() self.raw = raw self.protocol = protocol self.host = host diff --git a/cterasdk/cio/core.py b/cterasdk/cio/core.py index 03bf17c7..536135f0 100644 --- a/cterasdk/cio/core.py +++ b/cterasdk/cio/core.py @@ -81,6 +81,7 @@ def instance(src, dest=None): return SrcDstParam.__instance def __init__(self, src, dest=None): + super().__init__() self._classname = self.__class__.__name__ self.src = src self.dest = dest @@ -97,6 +98,7 @@ def instance(): return ActionResourcesParam.__instance def __init__(self): + super().__init__() self._classname = self.__class__.__name__ self.urls = [] ActionResourcesParam.__instance = self # pylint: disable=unused-private-member @@ -115,6 +117,7 @@ def instance(path, access, expire_on): return CreateShareParam.__instance def __init__(self, path, access, expire_on): + super().__init__() self._classname = self.__class__.__name__ self.url = path self.share = Object() @@ -131,6 +134,7 @@ def __init__(self, path, access, expire_on): class FetchResourcesParam(Object): def __init__(self): + super().__init__() self._classname = 'FetchResourcesParam' self.start = 0 self.limit = 100 diff --git a/cterasdk/clients/errors.py b/cterasdk/clients/errors.py index 7d960e39..d51496aa 100644 --- a/cterasdk/clients/errors.py +++ b/cterasdk/clients/errors.py @@ -8,6 +8,7 @@ class ClientError(Object): def __init__(self, exception, message): + super().__init__() self.request = Object() self.request.method = exception.request_info.method self.request.url = str(exception.request_info.real_url) diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index 7cc65552..312547b9 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -7,6 +7,7 @@ class Object(MutableMapping): # pylint: disable=too-many-instance-attributes def __init__(self, **kwargs): + super().__init__() for k, v in kwargs.items(): setattr(self, k, v) diff --git a/cterasdk/common/types.py b/cterasdk/common/types.py index c67dad5b..145d3e6f 100644 --- a/cterasdk/common/types.py +++ b/cterasdk/common/types.py @@ -28,6 +28,7 @@ def convert(rule, classname, property_name, assignment=None): class Operator(Object): def __init__(self, right): + super().__init__() self._classname = self.__class__.__name__ # pylint: disable=protected-access self.right = right @@ -75,6 +76,7 @@ class AfterOperator(Operator): class AdvancedFilterRule(Object): def __init__(self, classname, field, operator): + super().__init__() self._classname = self.__class__.__name__ # pylint: disable=protected-access self.className = classname self.fieldName = field @@ -283,6 +285,7 @@ def root(included): class FileEntry(Object): def __init__(self, name, display_name=None, included=None): + super().__init__() self._classname = self.__class__.__name__ # pylint: disable=protected-access self.name = name self.displayName = display_name @@ -300,6 +303,7 @@ class BackupSet(Object): def __init__(self, name, directory_tree=None, filter_rules=None, defaults_dirs=None, template_dirs=None, enabled=True, boolean_function=None, comment=None): + super().__init__() self._classname = self.__class__.__name__ # pylint: disable=protected-access self.name = name self.isEnabled = enabled @@ -334,6 +338,7 @@ def __init__(self, apps): class TaskSchedule(Object): def __init__(self): + super().__init__() self._classname = 'TaskSchedule' # pylint: disable=protected-access self.mode = None @@ -447,6 +452,7 @@ class ADDomainIDMapping(Object): :param int end: The maximum id to use for mapping """ def __init__(self, domain, start, end): + super().__init__() self._classname = 'ADDomainIDMapping' self.domainFlatName = domain self.minID = start @@ -499,6 +505,7 @@ def build(self): class SoftwareUpdatesTopic(Object): def __init__(self, enabled, reboot_after_update, reboot_when): + super().__init__() self._classname = "SoftwareUpdatesSettings" self.enabled = enabled if enabled else None self.rebootAfterUpdate = reboot_after_update if reboot_after_update else None diff --git a/cterasdk/common/utils.py b/cterasdk/common/utils.py index 203c1958..ac5ea1da 100644 --- a/cterasdk/common/utils.py +++ b/cterasdk/common/utils.py @@ -116,6 +116,7 @@ def __init__(self, uid, tenant=None, classname=None, name=None, more=None): :param str,optional name: Base object name :param str,optional more: Base object more info """ + super().__init__() self.uid = uid self.tenant = tenant self.classname = classname diff --git a/cterasdk/core/query.py b/cterasdk/core/query.py index ad75749a..7bbed9bf 100644 --- a/cterasdk/core/query.py +++ b/cterasdk/core/query.py @@ -80,12 +80,14 @@ def fromValue(value, ref): class Filter(Object): def __init__(self, field): + super().__init__() self.field = field class FilterBuilder(Object): def __init__(self, name, reference=False): + super().__init__() self.filter = Filter(name) self.reference = reference @@ -145,6 +147,7 @@ def setValue(self, value): class QueryParams(Object): def __init__(self): + super().__init__() self.startFrom = 0 self.countLimit = 50 diff --git a/cterasdk/core/types.py b/cterasdk/core/types.py index aa3b24b9..c6222fab 100644 --- a/cterasdk/core/types.py +++ b/cterasdk/core/types.py @@ -428,6 +428,7 @@ def secondary(self): class AccessControlRule(Object): def __init__(self, group, role): + super().__init__() self._classname = 'AccessControlRule' self.group = group self.role = role @@ -610,6 +611,7 @@ def build(self): class Task(Object): def __init__(self, task_id, name): + super().__init__() self.id = task_id self.name = name @@ -701,6 +703,7 @@ def build(self): class ExtendedAttribute(Object): def __init__(self, name, supported): + super().__init__() self._classname = 'ExtendedAttributesInfo' # pylint: disable=protected-access self.name = name self.supported = supported @@ -758,6 +761,7 @@ class RoleSettings(Object): # pylint: disable=too-many-instance-attributes def __init__(self, name, sudo, enable_remote_wipe, enable_sso, enable_seeding_export, enable_seeding_import, access_end_user_folders, update_settings, update_roles, update_account_emails, update_account_password, manage_cloud_drives, manage_plans, manage_users, manage_logs, allow_folders_files_permanent_delete, can_manage_legal_holds, can_manage_compliance_settings): + super().__init__() self.name = name self.sudo = sudo self.enable_remote_wipe = enable_remote_wipe diff --git a/cterasdk/direct/types.py b/cterasdk/direct/types.py index c22650e1..b6fc3167 100644 --- a/cterasdk/direct/types.py +++ b/cterasdk/direct/types.py @@ -209,6 +209,7 @@ class ChunkMetadata(Object): :ivar int length: Part Length """ def __init__(self, url, index, offset, length): + super().__init__() self.url = url self.index = index self.offset = offset @@ -221,5 +222,6 @@ class FileMetadata(Object): """ def __init__(self, f): + super().__init__() self.encryption_key = utils.utf8_decode(base64.b64encode(f.encryption_key)) self.chunks = [ChunkMetadata(chunk.url, chunk.index, chunk.offset, chunk.length) for chunk in f.chunks] diff --git a/cterasdk/edge/ctera_migrate.py b/cterasdk/edge/ctera_migrate.py index 71023b0f..dead3d7a 100644 --- a/cterasdk/edge/ctera_migrate.py +++ b/cterasdk/edge/ctera_migrate.py @@ -247,6 +247,7 @@ class Task(Object): """Class representing a migration tool task""" def __init__(self, task_id, task_type, name, created_at=None, source=None, source_type=None, last_status=None, shares=None, notes=None): + super().__init__() self.id = task_id self.type = {v: k for k, v in TaskType.__dict__.items() if not k.startswith('_')}.get(task_type).lower() self.name = name diff --git a/cterasdk/edge/query.py b/cterasdk/edge/query.py index 7add202f..4efe1389 100644 --- a/cterasdk/edge/query.py +++ b/cterasdk/edge/query.py @@ -43,6 +43,7 @@ def iterator(edge, path, param=None, name=None, callback_response=None): class QueryParam(Object): def __init__(self): + super().__init__() self.startFrom = 0 self.countLimit = 50 diff --git a/cterasdk/edge/types.py b/cterasdk/edge/types.py index d32d764d..fe2f1149 100644 --- a/cterasdk/edge/types.py +++ b/cterasdk/edge/types.py @@ -302,6 +302,7 @@ class DeduplicationStatus(Object): """ def __init__(self, size, usage): + super().__init__() self.size = size self.usage = usage diff --git a/cterasdk/lib/session/base.py b/cterasdk/lib/session/base.py index 329a2b37..3cbfe1c5 100644 --- a/cterasdk/lib/session/base.py +++ b/cterasdk/lib/session/base.py @@ -10,6 +10,7 @@ class BaseUser(Object): """Base User Account""" def __init__(self, name, domain=None): + super().__init__() self.name = name self.domain = domain @@ -32,6 +33,7 @@ def __init__(self, address, product): :param str address: Hostname or IP address :param cterasdk.lib.session.types.Product product: Product """ + super().__init__() self.address = address self.product = product self.connection_status = ConnectionStatus.Disconnected diff --git a/cterasdk/lib/session/edge.py b/cterasdk/lib/session/edge.py index 6e3ec161..4e351c92 100644 --- a/cterasdk/lib/session/edge.py +++ b/cterasdk/lib/session/edge.py @@ -9,6 +9,7 @@ class Connection(Object): """Connection""" def __init__(self, remote, source=None): + super().__init__() self.remote = remote if source: self.source = source From 11e4dd85ac9b23bf74739c0963e52091409e3184 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Wed, 28 May 2025 22:49:47 -0400 Subject: [PATCH 4/6] update device object --- cterasdk/common/object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index 312547b9..65fd324b 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -7,7 +7,6 @@ class Object(MutableMapping): # pylint: disable=too-many-instance-attributes def __init__(self, **kwargs): - super().__init__() for k, v in kwargs.items(): setattr(self, k, v) @@ -36,6 +35,7 @@ def __repr__(self): class Device(Object): def __init__(self, uid, version, firmware): + super().__init__() self.namespace = 'http://www.w3.org/2001/XMLSchema-instance' self.location = '../../db/resources/db.xsd' self.id = uid From 20ecd1942aaf8c680f67b8daf53a06d455230417 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Fri, 30 May 2025 15:19:50 -0400 Subject: [PATCH 5/6] support 8.3 portal direct-io api and improve log messages --- cterasdk/direct/client.py | 34 ++++--- cterasdk/direct/exceptions.py | 6 +- cterasdk/direct/lib.py | 85 ++++++++++++------ cterasdk/direct/types.py | 161 ++++++++++++---------------------- 4 files changed, 142 insertions(+), 144 deletions(-) diff --git a/cterasdk/direct/client.py b/cterasdk/direct/client.py index 0933e3a7..5e63c447 100644 --- a/cterasdk/direct/client.py +++ b/cterasdk/direct/client.py @@ -4,7 +4,7 @@ from . import filters from .credentials import KeyPair, Bearer from .lib import get_chunks, decrypt_encryption_key, process_chunks -from .types import File, ByteRange, FileMetadata +from .types import ByteRange from .stream import Streamer from ..objects.endpoints import DefaultBuilder, EndpointBuilder @@ -23,10 +23,15 @@ def __init__(self, baseurl, credentials): self._client = AsyncClient(DefaultBuilder(), settings=cterasdk.settings.io.direct.storage.settings, authenticator=lambda *_: True) self._credentials = credentials - async def _direct(self, file_id): - server_object = await get_chunks(self._api, self._credentials, file_id) - encryption_key = decrypt_encryption_key(file_id, server_object.wrapped_key, self._credentials.secret_access_key) - return File(file_id, encryption_key, server_object.chunks) + async def _chunks(self, file_id): + metadata = await get_chunks(self._api, self._credentials, file_id) + if metadata.encrypted: + metadata.encryption_key = decrypt_encryption_key( + metadata.file_id, + metadata.encryption_key, + self._credentials.secret_access_key + ) + return metadata async def metadata(self, file_id): """ @@ -34,7 +39,8 @@ async def metadata(self, file_id): :param int file_id: File ID. """ - return FileMetadata(await self._direct(file_id)) + meta = await self._chunks(file_id) + return meta.serialize() async def blocks(self, file_id, blocks, max_workers): """ @@ -47,8 +53,8 @@ async def blocks(self, file_id, blocks, max_workers): :returns: List of Blocks. :rtype: list[cterasdk.direct.types.Block] """ - file = await self._direct(file_id) - executor = await self.executor(filters.blocks(file, blocks), file.encryption_key, file_id, max_workers) + meta = await self._chunks(file_id) + executor = self.executor(filters.blocks(meta, blocks), meta.encryption_key, meta.file_id, max_workers) return await executor() async def streamer(self, file_id, byte_range): @@ -60,13 +66,13 @@ async def streamer(self, file_id, byte_range): :returns: Streamer Object :rtype: cterasdk.direct.stream.Streamer """ - file = await self._direct(file_id) + meta = await self._chunks(file_id) byte_range = byte_range if byte_range is not None else ByteRange.default() max_workers = cterasdk.settings.sessions.ctera_direct.streamer.max_workers - executor = await self.executor(filters.span(file, byte_range), file.encryption_key, file_id, max_workers) + executor = self.executor(filters.span(meta, byte_range), meta.encryption_key, file_id, max_workers) return Streamer(executor, byte_range) - async def executor(self, chunks, encryption_key, file_id=None, max_workers=None): + def executor(self, chunks, encryption_key, file_id=None, max_workers=None): """ Get Blocks. @@ -142,6 +148,12 @@ async def streamer(self, file_id, byte_range=None): """ return await self._client.streamer(file_id, byte_range) + def executor(self, chunks, encryption_key, max_workers): + """ + Get download executor for download from chunk metadata + """ + return self._client.executor(chunks, encryption_key, None, max_workers) + async def close(self): await self._client.close() diff --git a/cterasdk/direct/exceptions.py b/cterasdk/direct/exceptions.py index 2e1a556f..c2c5f1ef 100644 --- a/cterasdk/direct/exceptions.py +++ b/cterasdk/direct/exceptions.py @@ -45,19 +45,19 @@ def __init__(self, filename): class BlocksNotFoundError(DirectIOAPIError): def __init__(self, filename): - super().__init__(errno.ENODATA, 'Blocks not found', filename) + super().__init__(errno.ENODATA, f'Could not find blocks for file ID: {filename}', filename) class BlockListConnectionError(DirectIOAPIError): def __init__(self, filename): - super().__init__(errno.ENETRESET, 'Failed to list blocks. Connection error', filename) + super().__init__(errno.ENETRESET, f'Failed to list blocks for file ID: {filename} due to a connection error', filename) class BlockListTimeout(DirectIOAPIError): def __init__(self, filename): - super().__init__(errno.ETIMEDOUT, 'Failed to list blocks. Timed out', filename) + super().__init__(errno.ETIMEDOUT, f'Timed out while listing blocks for file ID: {filename}', filename) class DecryptKeyError(DirectIOError): diff --git a/cterasdk/direct/lib.py b/cterasdk/direct/lib.py index 7adeeb15..6a0318d3 100644 --- a/cterasdk/direct/lib.py +++ b/cterasdk/direct/lib.py @@ -1,7 +1,7 @@ import logging import asyncio -from .types import DirectIOResponse, Block +from .types import Metadata, Block from .credentials import KeyPair, Bearer from .crypto import decrypt_key, decrypt_block from .decompressor import decompress @@ -32,7 +32,7 @@ async def retry(coro, retries=3, backoff=1): if attempts == retries: raise error wait = backoff * (2 ** (attempts - 1)) - logger.debug('Failed attempt number %s. Retrying in %s seconds.', attempts, wait) + logger.debug('Download of block failed on attempt %s. Retrying in %s seconds...', attempts, wait) await asyncio.sleep(wait) @@ -46,23 +46,51 @@ async def get_object(client, file_id, chunk): :rtype: bytes """ async def get_object_coro(): - parameters = {'file_id': file_id, 'number': chunk.index, 'offset': chunk.offset} - logger.debug('Downloading Block. %s', parameters) + + message = ( + f"Downloading block #{chunk.index} " + f"(offset={chunk.offset}, length={chunk.length})" + ) + + if file_id: + message += f" for file ID {file_id}" + + error_message, exception = None, None + + logger.debug(message) try: response = await client.get(chunk.url) return await response.read() except ConnectionError: - logger.error('Failed to download block. Connection error. %s', parameters) - raise DownloadConnectionError(file_id, chunk) + error_message = 'connection' + exception = DownloadConnectionError(file_id, chunk) except asyncio.TimeoutError: - logger.error('Failed to download block. Timed out. %s', parameters) - raise DownloadTimeout(file_id, chunk) + error_message = 'timeout' + exception = DownloadTimeout(file_id, chunk) except IOError as error: - logger.error('Failed to download block. IO Error. %s', parameters) - raise DownloadError(error, file_id, chunk) + error_message = 'io' + exception = DownloadError(error, file_id, chunk) except ClientResponseException as error: - logger.error('Failed to download block. Error. %s', parameters) - raise DownloadError(error.response, file_id, chunk) + error_message = 'unknown' + exception = DownloadError(error.response, file_id, chunk) + + error_messages = { + "connection": "Connection error", + "timeout": "Timed out", + "io": "I/O error", + "unknown": "Unknown error" + } + + message = ( + f"Failed to download block #{chunk.index} " + f"(offset={chunk.offset}, length={chunk.length})" + ) + if file_id: + message = message + f" for file ID {file_id}" + + message = message + f": {error_messages.get(error_message, 'Unknown error')}." + logger.error(message) + raise exception return await retry(get_object_coro) @@ -118,9 +146,13 @@ async def process_chunk(client, file_id, chunk, encryption_key, semaphore): :rtype: cterasdk.direct.types.Block """ async def process(client, chunk, encryption_key): - parameters = {'file_id': file_id, 'number': chunk.index, 'offset': chunk.offset} - logger.debug('Processing Block. %s', parameters) - + message = ( + f"Processing block {chunk.index} " + f"(offset={chunk.offset}, length={chunk.length})" + ) + if file_id: + message = message + f" for file ID {file_id}" + logger.debug(message) encrypted_object = await get_object(client, file_id, chunk) decrypted_object = await decrypt_object(file_id, encrypted_object, encryption_key, chunk) decompressed_object = await decompress_object(file_id, decrypted_object, chunk) @@ -144,10 +176,12 @@ async def process_chunks(client, file_id, chunks, encryption_key, semaphore=None :returns: List of futures. :rtype: list[asyncio.Task] """ - parameters = {'file_id': file_id, 'blocks': len(chunks)} + message = [f"Processing {len(chunks)} blocks"] + if file_id: + message.append(f"for file ID {file_id}") if semaphore: - parameters['max_workers'] = semaphore._value # pylint: disable=protected-access - logger.debug('Processing Blocks. %s', parameters) + message.append(f"using up to {semaphore._value} workers") + logger.debug(' '.join(message)) futures = [] for chunk in chunks: futures.append(asyncio.create_task(process_chunk(client, file_id, chunk, encryption_key, semaphore))) @@ -182,9 +216,11 @@ def create_authorization_header(credentials): authorization_header = None if isinstance(credentials, Bearer): + logger.debug('Initializing client using bearer token') authorization_header = f'Bearer {credentials.bearer}' elif isinstance(credentials, KeyPair): + logger.debug('Initializing client using key pair.') authorization_header = f'Bearer {credentials.access_key_id}' return {'Authorization': authorization_header} @@ -197,17 +233,16 @@ async def get_chunks(api, credentials, file_id): :param cterasdk.clients.clients.AsyncJSON api: Asynchronous JSON Client. :param int file_id: File ID. :returns: Wrapped key and file chunks. - :rtype: cterasdk.direct.types.DirectIOResponse + :rtype: cterasdk.direct.types.Metadata """ async def get_chunks_coro(): - parameters = {'file_id': file_id} - logger.debug('Listing blocks. %s', parameters) + logger.debug('Listing blocks for file ID: %s', file_id) try: response = await api.get(f'{file_id}', headers=create_authorization_header(credentials)) if not response.chunks: - logger.error('Blocks not found. %s', parameters) + logger.error('Could not find blocks for file ID: %s.', file_id) raise BlocksNotFoundError(file_id) - return DirectIOResponse(response) + return Metadata(file_id, response) except ClientResponseException as error: if error.response.status == 400: raise NotFoundError(file_id) @@ -217,10 +252,10 @@ async def get_chunks_coro(): raise UnprocessableContent(file_id) raise error except ConnectionError: - logger.error('Failed to list blocks. Connection error. %s', parameters) + logger.error('Failed to list blocks for file ID: %s due to a connection error.', file_id) raise BlockListConnectionError(file_id) except asyncio.TimeoutError: - logger.error('Failed to list blocks. Timed out. %s', parameters) + logger.error('Timed out while listing blocks for file ID: %s.', file_id) raise BlockListTimeout(file_id) return await retry(get_chunks_coro) diff --git a/cterasdk/direct/types.py b/cterasdk/direct/types.py index b6fc3167..e6dfb9b7 100644 --- a/cterasdk/direct/types.py +++ b/cterasdk/direct/types.py @@ -1,3 +1,4 @@ +import copy import base64 from ..common import Object, utils @@ -40,20 +41,63 @@ def default(): return ByteRange(0) -class DirectIOResponse: +class CompressionLib: + """ + Compression Library + + :ivar str Snappy: Snappy + :ivar int Gzip: Gzip + :ivar int Off: No Compression + """ + Snappy = 'SNAPPY' + Gzip = 'GZIP' + Off = 'NONE' + + +class Chunk(Object): + + def __init__(self, index, offset, url, length): + """ + Initialize a Chunk. + + :param int index: Chunk index. + :param int offset: Chunk offset. + :param str url: Signed URL. + :param int length: Object length. + """ + super().__init__( + index=index, + offset=offset, + url=url, + length=length + ) + + +class Metadata(Object): + """ + CTERA Direct IO File Metadata + """ - def __init__(self, server_object): + def __init__(self, file_id, server_object): """ - Initialize a Get Response Object. + Initialize a Direct IO metadata response object. :param int file_id: File ID. :param cterasdk.common.object.Object server_object: Response Object. """ - self._wrapped_key = server_object.wrapped_key - self._chunks = DirectIOResponse._create_chunks(server_object.chunks) + super().__init__( + file_id=file_id, + encrypted = server_object.encrypt_info.data_encrypted, + compressed = server_object.compression_type != CompressionLib.Off, + chunks = Metadata._format_chunks(server_object.chunks) + ) + self.encryption_key = server_object.encrypt_info.wrapped_key if self.encrypted else None + self.compression_library = server_object.compression_type if self.compressed else None + last_chunk = self.chunks[-1] + self.size = last_chunk.offset + last_chunk.length @staticmethod - def _create_chunks(server_object): + def _format_chunks(server_object): """ Create Chunks. @@ -70,79 +114,14 @@ def _create_chunks(server_object): offset = offset + chunk.len return chunks - @property - def wrapped_key(self): - return self._wrapped_key - - @property - def chunks(self): - return self._chunks - - -class Chunk: - """Chunk to Retrieve""" - - def __init__(self, index, offset, url, length): - """ - Initialize a Chunk. - - :param int index: Chunk index. - :param int offset: Chunk offset. - :param str url: Signed URL. - :param int length: Object length. - """ - self._index = index - self._offset = offset - self._url = url - self._length = length - - @property - def index(self): - return self._index - - @property - def offset(self): - return self._offset - - @property - def url(self): - return self._url - - @property - def length(self): - return self._length - - -class File: - - def __init__(self, file_id, encryption_key, chunks): + def serialize(self): """ - Initialize a File Object. - - :param int file_id: File ID. - :param str encryption_key: Encryption Key. - :param cterasdk.direct.types.Chunk chunks: List of Chunks. + Serialize Direct IO metadata to a dictionary. """ - self._file_id = file_id - self._encryption_key = encryption_key - self._chunks = chunks - - @property - def file_id(self): - return self._file_id - - @property - def encryption_key(self): - return self._encryption_key - - @property - def chunks(self): - return self._chunks - - @property - def size(self): - last_chunk = self._chunks[-1] - return last_chunk.offset + last_chunk.length + x = copy.deepcopy(self) + if self.encrypted: + x.encryption_key = utils.utf8_decode(base64.b64encode(self.encryption_key)) + return x class Block: @@ -197,31 +176,3 @@ def fragment(self, byte_range): data = self._data[start:end] return Block(self._file_id, self._number, self._offset + start if start else self._offset, data, len(data)) - - -class ChunkMetadata(Object): - """ - Direct IO File Chunk Metadata Object - - :ivar str url: Part URL - :ivar int index: Part Index - :ivar int offset: Part Offset - :ivar int length: Part Length - """ - def __init__(self, url, index, offset, length): - super().__init__() - self.url = url - self.index = index - self.offset = offset - self.length = length - - -class FileMetadata(Object): - """ - Direct IO File Metadata Object - """ - - def __init__(self, f): - super().__init__() - self.encryption_key = utils.utf8_decode(base64.b64encode(f.encryption_key)) - self.chunks = [ChunkMetadata(chunk.url, chunk.index, chunk.offset, chunk.length) for chunk in f.chunks] From 0778306c2207c3bc8455cb40ec0e6480288436db Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Fri, 30 May 2025 15:42:21 -0400 Subject: [PATCH 6/6] update to pass lint and ut --- cterasdk/direct/lib.py | 2 +- cterasdk/direct/types.py | 6 +++--- tests/ut/aio/direct/test_get_metadata.py | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cterasdk/direct/lib.py b/cterasdk/direct/lib.py index 6a0318d3..a7e71b05 100644 --- a/cterasdk/direct/lib.py +++ b/cterasdk/direct/lib.py @@ -180,7 +180,7 @@ async def process_chunks(client, file_id, chunks, encryption_key, semaphore=None if file_id: message.append(f"for file ID {file_id}") if semaphore: - message.append(f"using up to {semaphore._value} workers") + message.append(f"using up to {semaphore._value} workers") # pylint: disable=protected-access logger.debug(' '.join(message)) futures = [] for chunk in chunks: diff --git a/cterasdk/direct/types.py b/cterasdk/direct/types.py index e6dfb9b7..d7061a4c 100644 --- a/cterasdk/direct/types.py +++ b/cterasdk/direct/types.py @@ -87,9 +87,9 @@ def __init__(self, file_id, server_object): """ super().__init__( file_id=file_id, - encrypted = server_object.encrypt_info.data_encrypted, - compressed = server_object.compression_type != CompressionLib.Off, - chunks = Metadata._format_chunks(server_object.chunks) + encrypted=server_object.encrypt_info.data_encrypted, + compressed=server_object.compression_type != CompressionLib.Off, + chunks=Metadata._format_chunks(server_object.chunks) ) self.encryption_key = server_object.encrypt_info.wrapped_key if self.encrypted else None self.compression_library = server_object.compression_type if self.compressed else None diff --git a/tests/ut/aio/direct/test_get_metadata.py b/tests/ut/aio/direct/test_get_metadata.py index 14efb80d..1f5f6b2a 100644 --- a/tests/ut/aio/direct/test_get_metadata.py +++ b/tests/ut/aio/direct/test_get_metadata.py @@ -31,7 +31,7 @@ async def test_get_file_metadata_not_found(self): with self.assertRaises(ctera_direct.exceptions.BlocksNotFoundError) as error: await self._direct.metadata(self._file_id) self.assertEqual(error.exception.errno, errno.ENODATA) - self.assertEqual(error.exception.strerror, 'Blocks not found') + self.assertEqual(error.exception.strerror, f'Could not find blocks for file ID: {self._file_id}') self.assertEqual(error.exception.filename, self._file_id) async def test_get_file_metadata_error_400(self): @@ -82,7 +82,8 @@ async def test_get_file_metadata_connection_error(self): with self.assertRaises(ctera_direct.exceptions.BlockListConnectionError) as error: await self._direct.metadata(self._file_id) self.assertEqual(error.exception.errno, errno.ENETRESET) - self.assertEqual(error.exception.strerror, 'Failed to list blocks. Connection error') + self.assertEqual(error.exception.strerror, + f'Failed to list blocks for file ID: {self._file_id} due to a connection error') self.assertEqual(error.exception.filename, self._file_id) async def test_get_file_metadata_timeout(self): @@ -91,7 +92,8 @@ async def test_get_file_metadata_timeout(self): with self.assertRaises(ctera_direct.exceptions.BlockListTimeout) as error: await self._direct.metadata(self._file_id) self.assertEqual(error.exception.errno, errno.ETIMEDOUT) - self.assertEqual(error.exception.strerror, 'Failed to list blocks. Timed out') + self.assertEqual(error.exception.strerror, + f'Timed out while listing blocks for file ID: {self._file_id}') self.assertEqual(error.exception.filename, self._file_id) @staticmethod