diff --git a/cterasdk/asynchronous/core/files/browser.py b/cterasdk/asynchronous/core/files/browser.py index 865c30d5..9c2b93f4 100644 --- a/cterasdk/asynchronous/core/files/browser.py +++ b/cterasdk/asynchronous/core/files/browser.py @@ -1,5 +1,6 @@ -from ....cio.core import CorePath +from ....cio.core import CorePath, a_await_or_future from ....lib.storage import asynfs, commonfs +from ....exceptions.io import FileConflict from ..base_command import BaseCommand from . import io @@ -61,11 +62,11 @@ async def download_many(self, target, objects, destination=None): handle = await self.handle_many(target, *objects) return await asynfs.write(directory, name, handle) - async def listdir(self, path, depth=None, include_deleted=False): + async def listdir(self, path=None, depth=None, include_deleted=False): """ List Directory - :param str path: Path + :param str,optional path: Path, defaults to the Cloud Drive root :param bool,optional include_deleted: Include deleted files, defaults to False """ return await io.listdir(self._core, self.normalize(path), depth=depth, include_deleted=include_deleted) @@ -105,18 +106,34 @@ async def public_link(self, path, access='RO', expire_in=30): """ return await io.public_link(self._core, self.normalize(path), access, expire_in) - async def copy(self, *paths, destination=None, wait=False): + async def _try_with_resolver(self, func, *paths, destination=None, resolver=None, cursor=None, wait=False): + async def wrapper(resume_from=None): + ref = await func(self._core, *paths, destination=destination, resolver=resolver, cursor=resume_from) + return await a_await_or_future(self._core, ref, wait) + + try: + return await wrapper(cursor) + except FileConflict as e: + if resolver: + return await wrapper(e.cursor) + raise + + async def copy(self, *paths, destination=None, resolver=None, cursor=None, wait=False): """ Copy one or more files or folders :param list[str] paths: List of paths :param str destination: Destination + :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` + :param cterasdk.common.object.Object cursor: Resume copy from cursor :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ try: - return await io.copy(self._core, *[self.normalize(path) for path in paths], destination=self.normalize(destination), wait=wait) + return await self._try_with_resolver(io.copy, *[self.normalize(path) for path in paths], + destination=self.normalize(destination), + resolver=resolver, cursor=cursor, wait=wait) except ValueError: raise ValueError('Copy destination was not specified.') @@ -188,7 +205,8 @@ async def rename(self, path, name, *, wait=False): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return await io.rename(self._core, self.normalize(path), name, wait=wait) + ref = await io.rename(self._core, self.normalize(path), name) + return await a_await_or_future(self._core, ref, wait) async def delete(self, *paths, wait=False): """ @@ -199,7 +217,8 @@ async def delete(self, *paths, wait=False): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return await io.remove(self._core, *[self.normalize(path) for path in paths], wait=wait) + ref = await io.remove(self._core, *[self.normalize(path) for path in paths]) + return await a_await_or_future(self._core, ref, wait) async def undelete(self, *paths, wait=False): """ @@ -210,19 +229,24 @@ async def undelete(self, *paths, wait=False): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return await io.recover(self._core, *[self.normalize(path) for path in paths], wait=wait) + ref = await io.recover(self._core, *[self.normalize(path) for path in paths]) + return await a_await_or_future(self._core, ref, wait) - async def move(self, *paths, destination=None, wait=False): + async def move(self, *paths, destination=None, resolver=None, cursor=None, wait=False): """ Move one or more files or folders :param list[str] paths: List of paths :param str destination: Destination + :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` + :param cterasdk.common.object.Object cursor: Resume copy from cursor :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ try: - return await io.move(self._core, *[self.normalize(path) for path in paths], destination=self.normalize(destination), wait=wait) + return await self._try_with_resolver(io.move, *[self.normalize(path) for path in paths], + destination=self.normalize(destination), + resolver=resolver, cursor=cursor, wait=wait) except ValueError: raise ValueError('Move destination was not specified.') diff --git a/cterasdk/asynchronous/core/files/io.py b/cterasdk/asynchronous/core/files/io.py index f391286f..9ea0c158 100644 --- a/cterasdk/asynchronous/core/files/io.py +++ b/cterasdk/asynchronous/core/files/io.py @@ -1,7 +1,7 @@ import logging -from ....cio.common import encode_request_parameter, a_await_or_future +from ....cio.common import encode_request_parameter from ....cio import core as fs -from ....exceptions.io import ResourceNotFoundError, NotADirectory, ResourceExistsError +from ....exceptions.io import ResourceNotFoundError, ResourceExistsError from .. import query @@ -68,40 +68,34 @@ async def makedirs(core, path): logger.debug('Resource already exists: %s', path.reference.as_posix()) -async def rename(core, path, name, *, wait=False): +async def rename(core, path, name): with fs.rename(path, name) as param: - ref = await core.v1.api.execute('', 'moveResources', param) - return await a_await_or_future(core, ref, wait) + return await core.v1.api.execute('', 'moveResources', param) -async def remove(core, *paths, wait=False): +async def remove(core, *paths): with fs.delete(*paths) as param: - ref = await core.v1.api.execute('', 'deleteResources', param) - return await a_await_or_future(core, ref, wait) + return await core.v1.api.execute('', 'deleteResources', param) -async def recover(core, *paths, wait=False): +async def recover(core, *paths): with fs.recover(*paths) as param: - ref = await core.v1.api.execute('', 'restoreResources', param) - return await a_await_or_future(core, ref, wait) + return await core.v1.api.execute('', 'restoreResources', param) -async def copy(core, *paths, destination=None, wait=False): - with fs.copy(*paths, destination=destination) as param: - ref = await core.v1.api.execute('', 'copyResources', param) - return await a_await_or_future(core, ref, wait) +async def copy(core, *paths, destination=None, resolver=None, cursor=None): + with fs.copy(*paths, destination=destination, resolver=resolver, cursor=cursor) as param: + return await core.v1.api.execute('', 'copyResources', param) -async def move(core, *paths, destination=None, wait=False): - with fs.move(*paths, destination=destination) as param: - ref = await core.v1.api.execute('', 'moveResources', param) - return await a_await_or_future(core, ref, wait) +async def move(core, *paths, destination=None, resolver=None, cursor=None): + with fs.move(*paths, destination=destination, resolver=resolver, cursor=cursor) as param: + return await core.v1.api.execute('', 'moveResources', param) async def ensure_directory(core, directory, suppress_error=False): present, resource = await metadata(core, directory, suppress_error=True) - if (not present or not resource.isFolder) and not suppress_error: - raise NotADirectory(directory.absolute) + fs.ensure_directory(present, resource, directory, suppress_error) return resource.isFolder if present else False, resource @@ -151,7 +145,9 @@ async def _validate_destination(core, name, destination): is_dir, resource = await ensure_directory(core, destination, suppress_error=True) if not is_dir: is_dir, resource = await ensure_directory(core, destination.parent) + fs.ensure_writeable(resource, destination.parent) return resource.cloudFolderInfo.uid, destination.name, destination.parent + fs.ensure_writeable(resource, destination) return resource.cloudFolderInfo.uid, name, destination diff --git a/cterasdk/asynchronous/core/login.py b/cterasdk/asynchronous/core/login.py index ea81b275..33dcaee7 100644 --- a/cterasdk/asynchronous/core/login.py +++ b/cterasdk/asynchronous/core/login.py @@ -1,7 +1,9 @@ import logging from .base_command import BaseCommand -from ...exceptions import CTERAException +from ...exceptions.transport import Forbidden +from ...exceptions.session import SessionExpired +from ...exceptions.auth import AuthenticationError logger = logging.getLogger('cterasdk.core') @@ -23,9 +25,9 @@ async def login(self, username, password): try: await self._core.v1.api.form_data('/login', {'j_username': username, 'j_password': password}) logger.info("User logged in. %s", {'host': host, 'user': username}) - except CTERAException: - logger.error("Login failed. %s", {'host': host, 'user': username}) - raise + except Forbidden as error: + logger.error('Login failed. %s', {'host': host, 'user': username}) + raise AuthenticationError() from error async def sso(self, ctera_ticket): """ @@ -40,5 +42,8 @@ async def logout(self): """ Log out of the portal """ - await self._core.v1.api.form_data('/logout', {}) - logger.info("User logged out. %s", {'host': self._core.host()}) + try: + await self._core.v1.api.form_data('/logout', {}) + logger.info("User logged out. %s", {'host': self._core.host()}) + except SessionExpired: + logger.info("Session expired and is no longer active.") diff --git a/cterasdk/asynchronous/core/notifications.py b/cterasdk/asynchronous/core/notifications.py index 1199095a..ee74d3a8 100644 --- a/cterasdk/asynchronous/core/notifications.py +++ b/cterasdk/asynchronous/core/notifications.py @@ -7,7 +7,7 @@ from ...common import Object from ...lib import CursorResponse from ...exceptions.transport import HTTPError -from ...exceptions.notifications import NotificationsError +from ...exceptions.notifications import NotificationsError, AncestorsError logger = logging.getLogger('cterasdk.notifications') @@ -86,12 +86,12 @@ async def ancestors(self, descendant): param = Object() param.folder_id = descendant.folder_id param.guid = descendant.guid - logger.debug('Getting ancestors. %s', {'guid': param.guid, 'folder_id': param.folder_id}) + logger.debug('Getting ancestors for: %s:%s', param.folder_id, param.guid) try: return await self._core.v2.api.post('/metadata/ancestors', param) - except HTTPError: - logger.error('Could not retrieve ancestors. %s', {'folder_id': param.folder_id, 'guid': param.guid}) - raise + except HTTPError as error: + logger.error('Could not retrieve ancestors for: %s:%s', param.folder_id, param.guid) + raise AncestorsError(param.folder_id, param.guid) from error class Service(BaseCommand): diff --git a/cterasdk/cio/common.py b/cterasdk/cio/common.py index 92cfce0a..0b517e74 100644 --- a/cterasdk/cio/common.py +++ b/cterasdk/cio/common.py @@ -77,27 +77,3 @@ def encode_request_parameter(param): return dict( inputXML=utf8_decode(toxmlstr(param)) ) - - -def await_or_future(ctera, ref, wait): - """ - Wait for task completion, or return an awaitable task object. - - :param str ref: Task reference - :param bool wait: ``True`` to wait for task completion, ``False`` to return an awaitable task object - """ - if wait: - return ctera.tasks.wait(ref) - return ctera.tasks.awaitable_task(ref) - - -async def a_await_or_future(ctera, ref, wait): - """ - Wait for task completion, or return an awaitable task object. - - :param str ref: Task reference - :param bool wait: ``True`` to wait for task completion, ``False`` to return an awaitable task object - """ - if wait: - return await ctera.tasks.wait(ref) - return ctera.tasks.awaitable_task(ref) diff --git a/cterasdk/cio/core.py b/cterasdk/cio/core.py index 3ea3f340..a426eb8c 100644 --- a/cterasdk/cio/core.py +++ b/cterasdk/cio/core.py @@ -3,11 +3,12 @@ from contextlib import contextmanager from ..objects.uri import quote, unquote from ..common import Object, DateTimeUtils -from ..core.enum import ProtectionLevel, CollaboratorType, SearchType, PortalAccountType, FileAccessMode, FileAccessError, \ - UploadError +from ..core.enum import ProtectionLevel, CollaboratorType, SearchType, PortalAccountType, FileAccessMode, \ + UploadError, ResourceAction, ResourceScope from ..core.types import PortalAccount, UserAccount, GroupAccount from ..exceptions.io import ResourceExistsError, PathValidationError, NameSyntaxError, \ - ReservedNameError, RestrictedRoot, InsufficientPermission + ReservedNameError, RestrictedRoot, PermissionDenied, FileConflict, RemoteStorageError, NotADirectory, \ + UnwriteableScope from ..exceptions.io import UploadException, OutOfQuota, RejectedByPolicy, NoStorageBucket, WindowsACLError from ..lib.iterator import DefaultResponse from . import common @@ -97,6 +98,13 @@ def __init__(self, src, dest=None): SrcDstParam.__instance = self # pylint: disable=unused-private-member +class ResourceActionCursor(Object): + + def __init__(self): + super().__init__() + self._classname = self.__class__.__name__ + + class ActionResourcesParam(Object): __instance = None @@ -110,11 +118,15 @@ def __init__(self): super().__init__() self._classname = self.__class__.__name__ self.urls = [] + self.startFrom = None ActionResourcesParam.__instance = self # pylint: disable=unused-private-member def add(self, param): self.urls.append(param) + def start_from(self, cursor): + self.startFrom = cursor + class CreateShareParam(Object): @@ -253,8 +265,20 @@ def recover(*paths): return _delete_or_recover(list(paths), message='Recovering item') -def _copy_or_move(paths, destination, *, message=None): +def _add_cursor_and_resolver(param, cursor, resolver): + if cursor: + param.startFrom = cursor + + if cursor and resolver: + if resolver.all: + param.startFrom.fileMoveConflictResolutaion = [resolver.build()] + else: + param.startFrom.skipHandler = resolver.handler + + +def _copy_or_move(paths, destination, resolver, cursor): param = ActionResourcesParam.instance() + _add_cursor_and_resolver(param, cursor, resolver) for path in paths: src, dest = path, destination if isinstance(path, tuple): @@ -264,18 +288,17 @@ def _copy_or_move(paths, destination, *, message=None): raise ValueError(f'Error: No destination specified for: {src}') dest = dest.join(src.name) param.add(SrcDstParam.instance(src=src.absolute_encode, dest=dest.absolute_encode)) - logger.info('%s from: %s to: %s', message, src.reference.as_posix(), dest.reference.as_posix()) yield param @contextmanager -def copy(*paths, destination=None): - return _copy_or_move(list(paths), destination, message='Copying item') +def copy(*paths, destination=None, resolver=None, cursor=None): + return _copy_or_move(list(paths), destination, resolver, cursor) @contextmanager -def move(*paths, destination=None): - return _copy_or_move(list(paths), destination, message='Moving item') +def move(*paths, destination=None, resolver=None, cursor=None): + return _copy_or_move(list(paths), destination, resolver, cursor) @contextmanager @@ -294,10 +317,22 @@ def handle(path): def destination_prerequisite_conditions(destination, name): - if not len(destination.reference.parts) > 0: - raise RestrictedRoot() if any(c in name for c in ['\\', '/', ':', '?', '&', '<', '>', '"', '|']): - raise NameSyntaxError() + raise NameSyntaxError(destination.join(name).reference.as_posix()) + + +def ensure_directory(present, resource, directory, suppress_error): + if (not present or not resource.isFolder) and not suppress_error: + raise NotADirectory(directory.reference.as_posix()) + + +def ensure_writeable(resource, directory): + if resource.scope == ResourceScope.Root: + raise RestrictedRoot() + if resource.scope not in [ResourceScope.Personal, ResourceScope.Project, ResourceScope.InsideCloudFolder]: + raise UnwriteableScope(directory.reference.as_posix(), resource.scope) + if resource.permission != FileAccessMode.RW: + raise PermissionDenied(directory.reference.as_posix(), ResourceAction.Write) @contextmanager @@ -531,29 +566,81 @@ def obtain_current_accounts(param): return current_accounts -def accept_error(error_type): +file_access_errors = { + "Conflict": FileConflict, + "PermissionDenied": PermissionDenied, + "DestinationNotExists": PathValidationError, + "FileWithTheSameNameExist": ResourceExistsError, + "InvalidName": NameSyntaxError, + "ReservedName": ReservedNameError +} + + +def await_or_future(core, ref, wait): + """ + Wait for task completion, or return an awaitable task object. + + :param str ref: Task reference + :param bool wait: ``True`` to wait for task completion, ``False`` to return an awaitable task object + """ + if wait: + task = core.tasks.wait(ref) + accept_error(task.error_type, **error_metadata(task)) + return task + return core.tasks.awaitable_task(ref) + + +async def a_await_or_future(ctera, ref, wait): + """ + Wait for task completion, or return an awaitable task object. + + :param str ref: Task reference + :param bool wait: ``True`` to wait for task completion, ``False`` to return an awaitable task object + """ + if wait: + task = await ctera.tasks.wait(ref) + accept_error(task.error_type, **error_metadata(task)) + return task + return ctera.tasks.awaitable_task(ref) + + +def error_metadata(task): + metadata = dict( + action=task.name + ) + if task.name in [ResourceAction.Copy, ResourceAction.Move]: + metadata.update(dict(cursor=task.cursor)) + if task.cursor.destResource: + metadata.update(dict(name=task.cursor.destResource.name)) + return metadata + + +def accept_error(error_type, **kwargs): """ Check if response contains an error. """ - error = { - FileAccessError.FileWithTheSameNameExist: ResourceExistsError(), - FileAccessError.DestinationNotExists: PathValidationError(), - FileAccessError.InvalidName: NameSyntaxError(), - FileAccessError.ReservedName: ReservedNameError(), - FileAccessError.PermissionDenied: InsufficientPermission() - }.get(error_type, None) - try: - if error: + if error_type not in [None, 'OK']: + exception_classname = file_access_errors.get(error_type, RemoteStorageError) + try: + raise exception_classname(**kwargs) + except ResourceExistsError as error: + logger.info('Resource already exists: a file or folder with this name already exists.') + raise error + except PathValidationError as error: + logger.error('Path validation failed: the specified destination path does not exist.') + raise error + except NameSyntaxError as error: + logger.error('Invalid name: the name contains characters that are not allowed.') + raise error + except ReservedNameError as error: + logger.error('Reserved name error: the name is reserved and cannot be used.') + raise error + except PermissionDenied as error: + logger.error('Permission denied: Inappropriate permissions to access this resource.') + raise error + except FileConflict as error: + logger.error('Conflict: a file with the same name already exists: %s', error.name) + raise error + except RemoteStorageError as error: + logger.error('An error occurred while performing operation.') raise error - except ResourceExistsError as error: - logger.info('Resource already exists: a file or folder with this name already exists.') - raise error - except PathValidationError as error: - logger.error('Path validation failed: the specified destination path does not exist.') - raise error - except NameSyntaxError as error: - logger.error('Invalid name: the name contains characters that are not allowed.') - raise error - except ReservedNameError as error: - logger.error('Reserved name error: the name is reserved and cannot be used.') - raise error diff --git a/cterasdk/cio/edge.py b/cterasdk/cio/edge.py index 2b3242e9..fabfd2a6 100644 --- a/cterasdk/cio/edge.py +++ b/cterasdk/cio/edge.py @@ -5,7 +5,8 @@ from ..common import Object from ..objects.uri import unquote from . import common -from ..exceptions.io import CTERAException, ResourceExistsError, RestrictedPathError +from ..exceptions.transport import HTTPError +from ..exceptions.io import ResourceExistsError, RestrictedPathError, RemoteStorageError logger = logging.getLogger('cterasdk.edge') @@ -67,9 +68,9 @@ def makedir(path): logger.info('Creating directory: %s', path.absolute) try: yield path.absolute - except CTERAException as error: + except HTTPError as error: try: - accept_error(error.response.message.msg, directory) + accept_error(error.error.response.error.msg, path=directory) except ResourceExistsError: logger.info('Directory already exists: %s', directory) logger.info('Directory created: %s', directory) @@ -126,17 +127,27 @@ def upload(name, destination, fd): yield param -def accept_error(response, reference): - error = { - "File exists": ResourceExistsError(), - "Creating a folder in this location is forbidden": RestrictedPathError(), - }.get(response, None) - try: - if error: +file_access_errors = { + "File exists": ResourceExistsError, + "Creating a folder in this location is forbidden": RestrictedPathError +} + + +def accept_error(error_message, **kwargs): + """ + Check if response contains an error. + """ + if error_message not in ['OK']: + exception_classname = file_access_errors.get(error_message, RemoteStorageError) + try: + if exception_classname: + raise exception_classname(**kwargs) + except ResourceExistsError as error: + logger.warning('Resource already exists: a file or folder with this name already exists.') + raise error + except RestrictedPathError as error: + logger.error('Creating a folder in the specified location is forbidden.') + raise error + except RemoteStorageError as error: + logger.error('An error occurred while performing operation.') raise error - except ResourceExistsError as error: - logger.warning('Resource already exists: a file or folder with this name already exists. %s', {'path': reference}) - raise error - except RestrictedPathError as error: - logger.error('Creating a folder in the specified location is forbidden. %s', {'name': reference}) - raise error diff --git a/cterasdk/clients/async_requests.py b/cterasdk/clients/async_requests.py index d52a2cd2..e030d0b5 100644 --- a/cterasdk/clients/async_requests.py +++ b/cterasdk/clients/async_requests.py @@ -4,6 +4,7 @@ import aiohttp from yarl import URL +from ..exceptions.transport import TLSError logger = logging.getLogger('cterasdk.http') @@ -45,7 +46,7 @@ async def _request(self, r, *, on_response=None): return asyncio.create_task(on_response(response)) except aiohttp.ClientSSLError as error: logger.warning(error) - raise + raise TLSError(error.host, error.port) from error except aiohttp.ClientProxyConnectionError as error: logger.warning(error) raise diff --git a/cterasdk/clients/clients.py b/cterasdk/clients/clients.py index d8b59834..850f9767 100644 --- a/cterasdk/clients/clients.py +++ b/cterasdk/clients/clients.py @@ -273,7 +273,7 @@ def propfind(self, path, depth, **kwargs): def mkcol(self, path, **kwargs): request = async_requests.MkcolRequest(self._builder(path), **kwargs) response = self.request(request, on_error=XMLHandler()) - return response.text() + return response.xml() @decorators.authenticated def copy(self, source, destination, *, overwrite=False, **kwargs): diff --git a/cterasdk/clients/errors.py b/cterasdk/clients/errors.py index 97d2c853..275a5e8d 100644 --- a/cterasdk/clients/errors.py +++ b/cterasdk/clients/errors.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from http import HTTPStatus from ..exceptions.transport import ( - BadRequest, Unauthorized, Forbidden, NotFound, Unprocessable, + BadRequest, Unauthorized, Forbidden, NotFound, NotAllowed, PreConditionFailed, Unprocessable, InternalServerError, BadGateway, ServiceUnavailable, GatewayTimeout, HTTPError ) from ..common import Object @@ -67,6 +67,8 @@ def raise_error(status, error): HTTPStatus.UNAUTHORIZED: Unauthorized, HTTPStatus.FORBIDDEN: Forbidden, HTTPStatus.NOT_FOUND: NotFound, + HTTPStatus.METHOD_NOT_ALLOWED: NotAllowed, + HTTPStatus.PRECONDITION_FAILED: PreConditionFailed, HTTPStatus.UNPROCESSABLE_ENTITY: Unprocessable, HTTPStatus.INTERNAL_SERVER_ERROR: InternalServerError, HTTPStatus.BAD_GATEWAY: BadGateway, diff --git a/cterasdk/common/object.py b/cterasdk/common/object.py index 8c641480..8cfeabf3 100644 --- a/cterasdk/common/object.py +++ b/cterasdk/common/object.py @@ -1,12 +1,23 @@ import re import json import logging +from datetime import datetime from collections.abc import MutableMapping logger = logging.getLogger('cterasdk.common') +class ObjectEncoder(json.JSONEncoder): + + def default(self, o): + if isinstance(o, datetime): + return o.isoformat() + if o.get('__dict__', None): + return o.__dict__ + return super().default(o) + + class Object(MutableMapping): # pylint: disable=too-many-instance-attributes def __init__(self, **kwargs): @@ -29,7 +40,7 @@ def __len__(self): return len(self.__dict__) def __str__(self): - return json.dumps(self, default=lambda o: o.__dict__, indent=5) + return json.dumps(self, cls=ObjectEncoder, indent=5) def __repr__(self): return str(self) diff --git a/cterasdk/core/enum.py b/cterasdk/core/enum.py index 0c4da94d..466a715f 100644 --- a/cterasdk/core/enum.py +++ b/cterasdk/core/enum.py @@ -646,12 +646,29 @@ class Reports: FolderGroups = 'folderGroupsStatisticsReport' +class ConflictHandler: + """ + Conflict Handler + + :ivar str Skip: Skip. + :ivar str Override: Override target. + :ivar str Rename: Append date to file name. + """ + Skip = 'Skip' + Override = 'Override' + Rename = 'Rename' + + class UploadError: """ Upload Error - :ivar QuotaViolation: User is out of quota. - :ivar RejectedByPolicy: Rejected by Cloud Drive policy rule. + :ivar str UserQuotaViolation: User is out of quota. + :ivar str FolderQuotaViolation: Directory is out of quota. + :ivar str PortalQuotaViolation: Team Portal is out of quota. + :ivar str RejectedByPolicy: Rejected by Cloud Drive policy rule. + :ivar str NoStorageBucket: No available storage bucket. + :ivar str WindowsACL: Illegal access to Windows ACL-enabled cloud drive folder. """ FolderQuotaViolation = 'Folder is out of quota' UserQuotaViolation = 'User is out of quota' @@ -661,48 +678,52 @@ class UploadError: WindowsACL = "Illegal access to NTACL folder" -class FileAccessError: - """ - File Access Error - - :ivar Conflict: Conflict occurred during file operation. - :ivar PermissionDenied: Operation denied due to insufficient permissions. - :ivar MoveDeletedFile: Attempted to move a file that was deleted. - :ivar CopyToSubFolder: Cannot copy a folder into one of its subfolders. - :ivar QuotaViolation: Operation exceeds allowed storage quota. - :ivar DestinationNotExists: Destination folder does not exist. - :ivar CancelledByUser: Operation was cancelled by the user. - :ivar InternalError: An internal error occurred. - :ivar UserPasswordRequired: User password is required to proceed. - :ivar PassphraseRequire: A passphrase is required. - :ivar CopyFileToRoot: Attempted to copy a file to the root directory. - :ivar FileWithTheSameNameExist: A file with the same name already exists. - :ivar RejectedByPolicy: Operation was rejected by a policy rule. - :ivar RejectedByWormSettings: Operation not allowed by WORM (Write Once Read Many) settings. - :ivar TooManyFailedAuthenticationAttemps: Too many failed authentication attempts. - :ivar UserActionTimeout: Timed out waiting for user action. - :ivar InvalidName: Provided name is invalid. - :ivar ReservedName: The name provided is reserved and cannot be used. - :ivar CannotRunPermanentDeleteWhenFsckIsRunning: FSCK is running, cannot perform permanent delete. - :ivar ResourceLocked: The resource is currently locked. - """ - Conflict = "Conflict" - PermissionDenied = "PermissionDenied" - MoveDeletedFile = "MoveDeletedFile" - CopyToSubFolder = "CopyToSubFolder" - QuotaViolation = "QuotaViolation" - DestinationNotExists = "DestinationNotExists" - CancelledByUser = "CancelledByUser" - InternalError = "InternalError" - UserPasswordRequired = "UserPasswordRequired" - PassphraseRequire = "PassphraseRequire" - CopyFileToRoot = "CopyFileToRoot" - FileWithTheSameNameExist = "FileWithTheSameNameExist" - RejectedByPolicy = "RejectedByPolicy" - RejectedByWormSettings = "RejectedByWormSettings" - TooManyFailedAuthenticationAttemps = "TooManyFailedAuthenticationAttemps" - UserActionTimeout = "UserActionTimeout" - InvalidName = "InvalidName" - ReservedName = "ReservedName" - CannotRunPermanentDeleteWhenFsckIsRunning = "CannotRunPermanentDeleteWhenFsckIsRunning" - ResourceLocked = "ResourceLocked" +class ResourceAction: + """ + Resource Action + + :ivar str Delete: Delete. + :ivar str Copy: Copy. + :ivar str Move: Move. + :ivar str Undelete: Undelete. + """ + Delete = 'Delete' + Undelete = 'Undelete' + Copy = 'Copy' + Move = 'Move' + Write = 'Write' + + +class ResourceScope: + """ + Resource Scope + + :ivar str Root: Root. + :ivar str ProjectsContainer: ProjectsContainer. + :ivar str Project: Project. + :ivar str SharedContainer: SharedContainer. + :ivar str SharedDomain: SharedDomain. + :ivar str Shared: Shared. + :ivar str BackupsContainer: BackupsContainer. + :ivar str Backup: Backup. + :ivar str Personal: Personal. + :ivar str UsersContainer: UsersContainer. + :ivar str UsersFoldersContainer: UsersFoldersContainer. + :ivar str CloudDrivesContainer: CloudDrivesContainer. + :ivar str OfflineFolder: OfflineFolder. + :ivar str InsideCloudFolder: InsideCloudFolder. + """ + Root = "Root" + ProjectsContainer = "ProjectsContainer" + Project = "Project" + SharedContainer = "SharedContainer" + SharedDomain = "SharedDomain" + Shared = "Shared" + BackupsContainer = "BackupsContainer" + Backup = "Backup" + Personal = "Personal" + UsersContainer = "UsersContainer" + UsersFoldersContainer = "UsersFoldersContainer" + CloudDrivesContainer = "CloudDrivesContainer" + OfflineFolder = "OfflineFolder" + InsideCloudFolder = "InsideCloudFolder" diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index 234c2d3e..67ef954f 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -1,7 +1,8 @@ import logging -from ...cio.core import CorePath -from ...exceptions import CTERAException +from ...cio.core import CorePath, await_or_future +from ...exceptions.transport import HTTPError +from ...exceptions.io import FileConflict from ...lib.storage import synfs, commonfs from ..base_command import BaseCommand from . import io @@ -67,11 +68,11 @@ def download_many(self, target, objects, destination=None): handle = self.handle_many(target, *objects) return synfs.write(directory, name, handle) - def listdir(self, path, depth=None, include_deleted=False): + def listdir(self, path=None, depth=None, include_deleted=False): """ List Directory - :param str path: Path + :param str,optional path: Path, defaults to the Cloud Drive root :param bool,optional include_deleted: Include deleted files, defaults to False """ return io.listdir(self._core, self.normalize(path), depth=depth, include_deleted=include_deleted) @@ -111,18 +112,34 @@ def public_link(self, path, access='RO', expire_in=30): """ return io.public_link(self._core, self.normalize(path), access, expire_in) - def copy(self, *paths, destination=None, wait=True): + def _try_with_resolver(self, func, *paths, destination=None, resolver=None, cursor=None, wait=True): + def wrapper(resume_from=None): + ref = func(self._core, *paths, destination=destination, resolver=resolver, cursor=resume_from) + return await_or_future(self._core, ref, wait) + + try: + return wrapper(cursor) + except FileConflict as e: + if resolver: + return wrapper(e.cursor) + raise + + def copy(self, *paths, destination=None, resolver=None, cursor=None, wait=True): """ Copy one or more files or folders :param list[str] paths: List of paths :param str destination: Destination + :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` + :param cterasdk.common.object.Object cursor: Resume copy from cursor :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ try: - return io.copy(self._core, *[self.normalize(path) for path in paths], destination=self.normalize(destination), wait=wait) + return self._try_with_resolver(io.copy, *[self.normalize(path) for path in paths], + destination=self.normalize(destination), + resolver=resolver, cursor=cursor, wait=wait) except ValueError: raise ValueError('Copy destination was not specified.') @@ -194,7 +211,8 @@ def rename(self, path, name, *, wait=True): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return io.rename(self._core, self.normalize(path), name, wait=wait) + ref = io.rename(self._core, self.normalize(path), name) + return await_or_future(self._core, ref, wait) def delete(self, *paths, wait=True): """ @@ -205,7 +223,8 @@ def delete(self, *paths, wait=True): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return io.remove(self._core, *[self.normalize(path) for path in paths], wait=wait) + ref = io.remove(self._core, *[self.normalize(path) for path in paths]) + return await_or_future(self._core, ref, wait) def undelete(self, *paths, wait=True): """ @@ -216,20 +235,25 @@ def undelete(self, *paths, wait=True): :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ - return io.recover(self._core, *[self.normalize(path) for path in paths], wait=wait) + ref = io.recover(self._core, *[self.normalize(path) for path in paths]) + return await_or_future(self._core, ref, wait) - def move(self, *paths, destination=None, wait=True): + def move(self, *paths, destination=None, resolver=None, cursor=None, wait=True): """ Move one or more files or folders :param list[str] paths: List of paths :param str destination: Destination + :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` + :param cterasdk.common.object.Object cursor: Resume copy from cursor :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. :returns: Task status object, or an awaitable task object :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` """ try: - return io.move(self._core, *[self.normalize(path) for path in paths], destination=self.normalize(destination), wait=wait) + return self._try_with_resolver(io.move, *[self.normalize(path) for path in paths], + destination=self.normalize(destination), + resolver=resolver, cursor=cursor, wait=wait) except ValueError: raise ValueError('Move destination was not specified.') @@ -297,6 +321,6 @@ def device_config(self, device, destination=None): try: destination = destination if destination is not None else f'{commonfs.downloads()}/{device}.xml' return self.download(f'backups/{device}/Device Configuration/db.xml', destination) - except CTERAException as error: - logger.error('Failed downloading configuration file. %s', {'device': device, 'error': error.response.reason}) + except HTTPError as error: + logger.error('Error downloading device configuration: %s. Reason: %s', device, error.response.reason) raise error diff --git a/cterasdk/core/files/io.py b/cterasdk/core/files/io.py index 6d04b97a..d8cfead5 100644 --- a/cterasdk/core/files/io.py +++ b/cterasdk/core/files/io.py @@ -1,7 +1,7 @@ import logging -from ...cio.common import encode_request_parameter, await_or_future +from ...cio.common import encode_request_parameter from ...cio import core as fs -from ...exceptions.io import ResourceNotFoundError, ResourceExistsError, NotADirectory +from ...exceptions.io import ResourceNotFoundError, ResourceExistsError from ...core import query from ..enum import CollaboratorType @@ -55,8 +55,9 @@ def walk(core, scope, path, include_deleted=False): def mkdir(core, path): with fs.makedir(path) as param: - response = core.api.execute('', 'makeCollection', param) - fs.accept_error(response) + error_type = core.api.execute('', 'makeCollection', param) + print(error_type) + fs.accept_error(error_type, path=path.reference.as_posix()) def makedirs(core, path): @@ -69,40 +70,34 @@ def makedirs(core, path): logger.debug('Resource already exists: %s', path.reference.as_posix()) -def rename(core, path, name, *, wait=True): +def rename(core, path, name): with fs.rename(path, name) as param: - ref = core.api.execute('', 'moveResources', param) - return await_or_future(core, ref, wait) + return core.api.execute('', 'moveResources', param) -def remove(core, *paths, wait=True): +def remove(core, *paths): with fs.delete(*paths) as param: - ref = core.api.execute('', 'deleteResources', param) - return await_or_future(core, ref, wait) + return core.api.execute('', 'deleteResources', param) -def recover(core, *paths, wait=True): +def recover(core, *paths): with fs.recover(*paths) as param: - ref = core.api.execute('', 'restoreResources', param) - return await_or_future(core, ref, wait) + return core.api.execute('', 'restoreResources', param) -def copy(core, *paths, destination=None, wait=True): - with fs.copy(*paths, destination=destination) as param: - ref = core.api.execute('', 'copyResources', param) - return await_or_future(core, ref, wait) +def copy(core, *paths, destination=None, resolver=None, cursor=None): + with fs.copy(*paths, destination=destination, resolver=resolver, cursor=cursor) as param: + return core.api.execute('', 'copyResources', param) -def move(core, *paths, destination=None, wait=True): - with fs.move(*paths, destination=destination) as param: - ref = core.api.execute('', 'moveResources', param) - return await_or_future(core, ref, wait) +def move(core, *paths, destination=None, resolver=None, cursor=None): + with fs.move(*paths, destination=destination, resolver=resolver, cursor=cursor) as param: + return core.api.execute('', 'moveResources', param) def ensure_directory(core, directory, suppress_error=False): present, resource = metadata(core, directory, suppress_error=True) - if (not present or not resource.isFolder) and not suppress_error: - raise NotADirectory(directory.absolute) + fs.ensure_directory(present, resource, directory, suppress_error) return resource.isFolder if present else False, resource @@ -152,7 +147,9 @@ def _validate_destination(core, name, destination): is_dir, resource = ensure_directory(core, destination, suppress_error=True) if not is_dir: is_dir, resource = ensure_directory(core, destination.parent) + fs.ensure_writeable(resource, destination.parent) return resource.cloudFolderInfo.uid, destination.name, destination.parent + fs.ensure_writeable(resource, destination) return resource.cloudFolderInfo.uid, name, destination diff --git a/cterasdk/core/login.py b/cterasdk/core/login.py index 3482c0f4..1405d774 100644 --- a/cterasdk/core/login.py +++ b/cterasdk/core/login.py @@ -2,6 +2,7 @@ from .base_command import BaseCommand from ..exceptions.transport import Forbidden +from ..exceptions.session import SessionExpired from ..exceptions.auth import AuthenticationError @@ -43,5 +44,8 @@ def logout(self): Log out of CTERA Portal """ username = self._core.session().account.name - self._core.api.form_data('/logout', {}) - logger.info("User logged out. %s", {'host': self._core.host(), 'user': username}) + try: + self._core.api.form_data('/logout', {}) + logger.info("User logged out. %s", {'host': self._core.host(), 'user': username}) + except SessionExpired: + logger.info("Session expired and is no longer active.") diff --git a/cterasdk/core/types.py b/cterasdk/core/types.py index c6222fab..ee655027 100644 --- a/cterasdk/core/types.py +++ b/cterasdk/core/types.py @@ -4,7 +4,7 @@ from ..lib.storage import commonfs from .enum import PortalAccountType, CollaboratorType, FileAccessMode, PlanCriteria, TemplateCriteria, \ - BucketType, LocationType, Platform, RetentionMode, Duration, ExtendedAttributes + BucketType, LocationType, Platform, RetentionMode, Duration, ExtendedAttributes, ConflictHandler CloudFSFolderFindingHelper = namedtuple('CloudFSFolderFindingHelper', ('name', 'owner')) @@ -827,3 +827,37 @@ def from_server_object(server_object): 'can_manage_compliance_settings': server_object.canManageComplianceSetting } return RoleSettings(**params) + + +class ConflictResolver: + + def __init__(self, handler, apply_all): + self._apply_all = apply_all + self._handler = handler + + @property + def all(self): + return self._apply_all + + @property + def handler(self): + return self._handler + + def build(self): + param = Object() + param._classname = 'FileMoveConflictResolutaion' # pylint: disable=protected-access + param.errorType = 'Conflict' + param.handler = self._handler + return param + + @staticmethod + def ignore(apply_all=True): + return ConflictResolver(ConflictHandler.Skip, apply_all) + + @staticmethod + def override(apply_all=True): + return ConflictResolver(ConflictHandler.Override, apply_all) + + @staticmethod + def rename(apply_all=True): + return ConflictResolver(ConflictHandler.Rename, apply_all) diff --git a/cterasdk/direct/client.py b/cterasdk/direct/client.py index 01d92faf..dc7012c7 100644 --- a/cterasdk/direct/client.py +++ b/cterasdk/direct/client.py @@ -56,7 +56,7 @@ async def blocks(self, file_id, byte_range=None, max_workers=None): :param int file_id: File ID. :param cterasdk.direct.types.ByteRange,optional byte_range: Byte Range. - :param int max_workers: Max concurrent tasks + :param int,optional max_workers: Max concurrent tasks :returns: List of Blocks. :rtype: list[cterasdk.direct.types.Block] """ @@ -65,18 +65,19 @@ async def blocks(self, file_id, byte_range=None, max_workers=None): executor = self.executor(filters.span(meta, byte_range), meta.encryption_key, meta.file_id, max_workers) return await executor() - async def streamer(self, file_id, byte_range=None): + async def streamer(self, file_id, byte_range=None, max_workers=None): """ Stream API. :param int file_id: File ID. :param cterasdk.direct.types.ByteRange,optional byte_range: Byte Range. + :param int,optional max_workers: Max concurrent tasks :returns: Streamer Object :rtype: cterasdk.direct.stream.Streamer """ meta = await self._chunks(file_id) byte_range = byte_range if byte_range is not None else ByteRange.default() - max_workers = cterasdk.settings.io.direct.streamer.max_workers + max_workers = max_workers if max_workers else cterasdk.settings.io.direct.streamer.max_workers executor = self.executor(filters.span(meta, byte_range), meta.encryption_key, file_id, max_workers) return Streamer(executor, byte_range) diff --git a/cterasdk/exceptions/io.py b/cterasdk/exceptions/io.py index ed7c34bf..6815a9d1 100644 --- a/cterasdk/exceptions/io.py +++ b/cterasdk/exceptions/io.py @@ -1,7 +1,7 @@ from .base import CTERAException -class RemoteStorageException(CTERAException): +class RemoteStorageError(CTERAException): """ Base Exception for Remote File Storage @@ -12,61 +12,78 @@ def __init__(self, message, path=None): self.path = path -class ResourceNotFoundError(RemoteStorageException): +class ResourceNotFoundError(RemoteStorageError): def __init__(self, path): super().__init__('Remote directory not found. Please verify the path and try again.', path) -class NotADirectory(RemoteStorageException): +class NotADirectory(RemoteStorageError): def __init__(self, path): super().__init__('Target validation error: Resource exists but it is not a directory.', path) -class ResourceExistsError(RemoteStorageException): +class ResourceExistsError(RemoteStorageError): - def __init__(self): - super().__init__('Resource already exists: a file or folder with this name already exists.') + def __init__(self, path): + super().__init__('Resource already exists: a file or folder with this name already exists.', path) -class PathValidationError(RemoteStorageException): +class PathValidationError(RemoteStorageError): - def __init__(self): - super().__init__('Path validation failed: the specified destination path does not exist.') + def __init__(self, path=None, **kwargs): # pylint: disable=unused-argument + super().__init__('Path validation failed: the specified destination path does not exist.', path) -class NameSyntaxError(RemoteStorageException): +class NameSyntaxError(RemoteStorageError): - def __init__(self): - super().__init__('Invalid name: the name contains characters that are not allowed "\\ / : ? & < > \" |".') + def __init__(self, path): + super().__init__('Invalid name: the name contains characters that are not allowed "\\ / : ? & < > \" |".', path) -class ReservedNameError(RemoteStorageException): +class ReservedNameError(RemoteStorageError): - def __init__(self): - super().__init__('Reserved name error: the name is reserved and cannot be used.') + def __init__(self, path): + super().__init__('Reserved name error: the name is reserved and cannot be used.', path) -class RestrictedPathError(RemoteStorageException): +class RestrictedPathError(RemoteStorageError): - def __init__(self): - super().__init__('Creating a folder in the specified location is forbidden.') + def __init__(self, path): + super().__init__('Creating a folder in the specified location is forbidden.', path) -class RestrictedRoot(RemoteStorageException): +class RestrictedRoot(RemoteStorageError): def __init__(self): super().__init__('Storing files to the root directory is forbidden.', '/') -class InsufficientPermission(RemoteStorageException): +class PermissionDenied(RemoteStorageError): - def __init__(self): - super().__init__('Permission denied: You must have appropriate permissions to access this resource.') + def __init__(self, action, path=None): + super().__init__('Permission denied: Inappropriate permissions to access this resource.', path) + self.action = action + + +class UnwriteableScope(RemoteStorageError): + + def __init__(self, path, scope): + super().__init__("Write access denied. This target is protected and cannot be modified.", path) + self.scope = scope + + +class FileConflict(RemoteStorageError): + + def __init__(self, action, name, cursor): + super().__init__('Conflict: a file with the same name already exists.') + self.action = action + self.name = name + self.cursor = cursor -class UploadException(RemoteStorageException): +class UploadException(RemoteStorageError): """ Upload Exception diff --git a/cterasdk/exceptions/notifications.py b/cterasdk/exceptions/notifications.py index c2e99ce1..7c2033e6 100644 --- a/cterasdk/exceptions/notifications.py +++ b/cterasdk/exceptions/notifications.py @@ -13,3 +13,16 @@ def __init__(self, cloudfolders, cursor): super().__init__('An error occurred while trying to retrieve notifications.') self.cloudfolders = cloudfolders self.cursor = cursor + + +class AncestorsError(CTERAException): + """ + Ancestors Error + + :ivar int folder_id: Cloud Drive folder unique identifer + :ivar str guid: File GUID + """ + def __init__(self, folder_id, guid): + super().__init__(f'Could not retrieve ancestors for: {folder_id}:{guid}') + self.folder_id = folder_id + self.guid = guid diff --git a/cterasdk/exceptions/session.py b/cterasdk/exceptions/session.py index 933f5346..185b87b6 100644 --- a/cterasdk/exceptions/session.py +++ b/cterasdk/exceptions/session.py @@ -5,7 +5,7 @@ class SessionExpired(CTERAException): """Session expiration""" def __init__(self): - super().__init__('Authentication error: Session expired.') + super().__init__('Authentication error: Session expired. Please log in again.') class NotLoggedIn(CTERAException): diff --git a/cterasdk/exceptions/transport.py b/cterasdk/exceptions/transport.py index cc7c6e2a..5e0ea324 100644 --- a/cterasdk/exceptions/transport.py +++ b/cterasdk/exceptions/transport.py @@ -19,117 +19,80 @@ def __init__(self, status, error): class BadRequest(HTTPError): - """ - Bad Request - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.BAD_REQUEST, error) class Unauthorized(HTTPError): - """ - Unauthorized - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.UNAUTHORIZED, error) class Forbidden(HTTPError): - """ - Unauthorized - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.FORBIDDEN, error) class NotFound(HTTPError): - """ - NotFound - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.NOT_FOUND, error) -class Unprocessable(HTTPError): - """ - Unprocessable +class NotAllowed(HTTPError): - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ + def __init__(self, error): + super().__init__(HTTPStatus.METHOD_NOT_ALLOWED, error) + + +class PreConditionFailed(HTTPError): + + def __init__(self, error): + super().__init__(HTTPStatus.PRECONDITION_FAILED, error) + + +class Unprocessable(HTTPError): def __init__(self, error): super().__init__(HTTPStatus.UNPROCESSABLE_ENTITY, error) class InternalServerError(HTTPError): - """ - InternalServerError - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.INTERNAL_SERVER_ERROR, error) class BadGateway(HTTPError): - """ - BadGateway - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.BAD_GATEWAY, error) class ServiceUnavailable(HTTPError): - """ - ServiceUnavailable - - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object - """ def __init__(self, error): super().__init__(HTTPStatus.SERVICE_UNAVAILABLE, error) class GatewayTimeout(HTTPError): + + def __init__(self, error): + super().__init__(HTTPStatus.GATEWAY_TIMEOUT, error) + + +class TLSError(CTERAException): """ - GatewayTimeout + TLS Error - :ivar int code: Status code - :ivar str name: Reason - :ivar cterasdk.clients.errors.Error error: Error object + :ivar str host: Host + :ivar int port: Port """ - def __init__(self, error): - super().__init__(HTTPStatus.GATEWAY_TIMEOUT, error) + def __init__(self, host, port): + super().__init__(f"TLS handshake to '{host}:{port}' failed.") + self.host = host + self.port = port diff --git a/cterasdk/lib/tasks.py b/cterasdk/lib/tasks.py index 30c262c9..94e55407 100644 --- a/cterasdk/lib/tasks.py +++ b/cterasdk/lib/tasks.py @@ -2,8 +2,10 @@ import time import logging import asyncio +from datetime import datetime from abc import ABC, abstractmethod +from ..common import Object from ..common.enum import TaskRunningStatus from ..exceptions.common import TaskWaitTimeoutError, AwaitableTaskException from ..exceptions.transport import HTTPError @@ -12,6 +14,80 @@ logger = logging.getLogger('cterasdk.common') +class BaseTask(Object): + """ + Base Task + + :ivar int id: Task ID + :ivar str name: Task name + :ivar datetime.datetime start_time: Start time + :ivar int elapsed_time: Elapsed time + :ivar cterasdk.common.enum.TaskRunningStatus status: Status + :ivar int percentage: Percentage + :ivar datetime.datetime end_time: End time + :ivar cterasdk.common.object.Object result: Task result + """ + def __init__(self, task): + super().__init__() + self.id = task.id + self.name = task.name + self.start_time = datetime.fromisoformat(task.startTime) + self.elapsed_time = task.elapsedTime + self.status = task.status + self.percentage = task.percentage + self.end_time = datetime.fromisoformat(task.endTime) if task.endTime is not None else None + self.result = task.result + + +class EdgeTask(BaseTask): + """ + Edge Task + + :ivar str description: Task description + """ + def __init__(self, task): + super().__init__(task) + self.description = task.description + + +class PortalTask(BaseTask): + """ + Portal Task + + :ivar str progress_str: Task progress description + :ivar int tenant: Tenant UID + """ + def __init__(self, task): + super().__init__(task) + self.progress_str = task.progstring + self.tenant = task.portalUid + + +class FilesystemTask(PortalTask): + """ + Filesystem Task + + :ivar int files_processed: Files processed + :ivar int bytes_processed: Bytes processed + :ivar int total_files: Total files + :ivar int total_bytes: Total bytes + :ivar str error_type: Task error + :ivar str file_in_progress: File in progress + :ivar int user_uid: User UID + :ivar cterasdk.common.object.Object cursor: Cursor + """ + def __init__(self, task): + super().__init__(task) + self.files_processed = task.filesProcessed + self.bytes_processed = task.bytesProcessed + self.total_files = task.totalFiles + self.total_bytes = task.totalBytes + self.error_type = task.errorType + self.file_in_progress = task.fileInProgress + self.user_uid = task.userUid + self.cursor = task.cursor + + class AwaitableTask(ABC): def __init__(self, ctera, ref): @@ -60,6 +136,10 @@ async def a_wait(self, timeout=None, poll_interval=None): def _task_reference(self, ref): raise NotImplementedError("Subclass must implement the '_task_reference' function.") + @abstractmethod + def _task_status_object(self, task): + raise NotImplementedError("Subclass must implement the '_task_status_object' function.") + @abstractmethod def status(self): raise NotImplementedError("Subclass must implement the 'status' function.") @@ -93,17 +173,20 @@ def _task_reference(self, ref): logger.error('Failed to parse task identifier from reference: %s', ref) raise ValueError(f'Failed to parse task identifier from reference: {ref}') + def _task_status_object(self, task): + return EdgeTask(task) + def status(self): """ Synchronous function to retrieve task status. """ - return self._ctera.api.get(self._ref) + return self._task_status_object(self._ctera.api.get(self._ref)) async def a_status(self): """ Asynchronous function to retrieve task status. """ - return await self._ctera.api.get(self._ref) + return self._task_status_object(await self._ctera.api.get(self._ref)) class AwaitablePortalTask(AwaitableTask): @@ -117,21 +200,26 @@ def _task_reference(self, ref): raise ValueError(f'Failed to parse task identifier from reference: {ref}') return match.group(0) + def _task_status_object(self, task): + if task._classname == 'FileManagerBgTask': # pylint: disable=protected-access + return FilesystemTask(task) + return PortalTask(task) + def status(self): """ Synchronous function to retrieve task status. """ if self._ctera.session().in_tenant_context(): - return self._ctera.api.execute('', 'getTaskStatus', self._ref) - return self._ctera.api.get(f'{self._ref}') + return self._task_status_object(self._ctera.api.execute('', 'getTaskStatus', self._ref)) + return self._task_status_object(self._ctera.api.get(f'{self._ref}')) async def a_status(self): """ Asynchronous function to retrieve task status. """ if self._ctera.session().in_tenant_context(): - return await self._ctera.v1.api.execute('', 'getTaskStatus', self._ref) - return await self._ctera.v1.api.get(f'{self._ref}') + return self._task_status_object(await self._ctera.v1.api.execute('', 'getTaskStatus', self._ref)) + return self._task_status_object(await self._ctera.v1.api.get(f'{self._ref}')) def _before_wait(timeout, poll_interval): diff --git a/docs/source/UserGuides/Miscellaneous/Changelog.rst b/docs/source/UserGuides/Miscellaneous/Changelog.rst index f2f4c245..da0ed0c8 100644 --- a/docs/source/UserGuides/Miscellaneous/Changelog.rst +++ b/docs/source/UserGuides/Miscellaneous/Changelog.rst @@ -4,11 +4,76 @@ Changelog 2.20.20 ------- +Related issues and pull requests on GitHub: `#316 `_, +`#317 `_ +`#318 `_ + + Improvements ^^^^^^^^^^^^ -* Add unique User-Agent header to all requests made by the CTERA Python SDK +* Added a unique ``User-Agent`` header to all requests made by the CTERA Python SDK +* Raised exceptions on upload errors to CTERA Portal +* Raised :py:class:`cterasdk.exceptions.session.SessionExpired` upon session expiration +* Listed the Cloud Drive root by default if no ``path`` argument was + provided to :py:func:`cterasdk.core.files.browser.FileBrowser.listdir` +* Added :py:class:`cterasdk.exceptions.notifications.AncestorsError` exception +* Added :py:class:`cterasdk.exceptions.transport.TLSError` exception +* Suppressed session expiration exceptions on logout +* Added support for resolving file conflicts on copy and move + operations using :py:class:`cterasdk.core.types.ConflictResolver` + +Bug Fixes +^^^^^^^^^ + +* Corrected Direct I/O object class references in the documentation + +.. code:: python + + """Catching upload exceptions""" + try: + ... + except cterasdk.exceptions.io.OutOfQuota as e: + print('Failure due to quota violation.') + except cterasdk.exceptions.io.RejectedByPolicy as e: + print('Failure due to Cloud Drive policy violation.') + except cterasdk.exceptions.io.NoStorageBucket as e: + print('No backend storage bucket is available.') + except cterasdk.exceptions.io.WindowsACLError as e: + print('Attempt to upload a file to a Windows ACL-enabled cloud drive folder.') + except cterasdk.exceptions.io.UploadException: + print('Base exception for any upload errors.') + + """Catching expired sessions""" + try: + ... + except cterasdk.exceptions.session.SessionExpired as e: + print('Session expired. Re-authenticate to establish a new session.') + +* Starting with this version, the CTERA Python SDK ``User-Agent`` header is formatted as follows: + +.. code:: + + CTERA Python SDK/2.20.20; aiohttp/3.9.5; (Windows 10; AMD64; Python 3.11.4); + +* Introduced support for resolving conflicts during copy and move operations + +.. code:: python + """Override destination on conflict""" + resolver = core_types.ConflictResolver.override() + user.files.copy(('My Files/Gelato.pptx', 'My Files/Slides/Gelato.pptx'), resolver=resolver) + + """Resume job from cursor""" + objects = ( + 'My Files/Gelato.pptx', 'My Files/Slides/Gelato.pptx', + 'Spreadsheets/Q1Summary.xlsx', 'Sheets/Q1Summary.pptx' + ) + try: + user.files.copy(objects) + except cterasdk.exceptions.io.FileConflict as e: + resolver = core_types.ConflictResolver.override() # override destination + user.files.copy(objects, resolver=resolver, cursor=e.cursor) # resume copy from cursor 2.20.19 ------- diff --git a/docs/source/UserGuides/Miscellaneous/Exceptions.rst b/docs/source/UserGuides/Miscellaneous/Exceptions.rst index 35a309a0..d24e06eb 100644 --- a/docs/source/UserGuides/Miscellaneous/Exceptions.rst +++ b/docs/source/UserGuides/Miscellaneous/Exceptions.rst @@ -37,7 +37,7 @@ Session I/O --- -.. autoclass:: cterasdk.exceptions.io.RemoteStorageException +.. autoclass:: cterasdk.exceptions.io.RemoteStorageError :noindex: :members: :show-inheritance: diff --git a/docs/source/UserGuides/Portal/Files.rst b/docs/source/UserGuides/Portal/Files.rst index 15c5efd9..b81c77b2 100644 --- a/docs/source/UserGuides/Portal/Files.rst +++ b/docs/source/UserGuides/Portal/Files.rst @@ -86,7 +86,6 @@ Versions .. code:: python - """When logged in as a Global Administrator""" versions = admin.files.versions('Users/John Smith/My Files/Documents') for version in versions: if not version.current: @@ -94,7 +93,6 @@ Versions print(version.calculatedTimestamp, item.name) - """When logged in as a Team Portal Administrator End User""" versions = user.files.versions('My Files/Documents') for version in versions: if not version.current: @@ -109,10 +107,8 @@ Download .. code:: python - """When logged in as a Global Administrator""" admin.files.download('Users/John Smith/My Files/Documents/Sample.docx') - """When logged in as a Team Portal Administrator End User""" user.files.download('My Files/Documents/Sample.docx') .. automethod:: cterasdk.core.files.browser.FileBrowser.download_many @@ -120,10 +116,8 @@ Download .. code:: python - """When logged in as a Global Administrator""" admin.files.download_many('Users/John Smith/My Files/Documents', ['Sample.docx', 'Wizard Of Oz.docx']) - """When logged in as a Team Portal Administrator End User""" user.files.download_many('My Files/Documents', ['Sample.docx', 'Wizard Of Oz.docx']) Copy @@ -132,9 +126,12 @@ Copy .. automethod:: cterasdk.core.files.browser.FileBrowser.copy :noindex: + To resolve file conflicts, use :py:class:`cterasdk.core.types.ConflictResolver` + .. code:: python - """When logged in as a Team Portal Administrator End User""" + admin.files.copy(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx'], destination='Users/John Smith/The/quick/brown/fox') + user.files.copy(*['My Files/Documents/Sample.docx', 'My Files/Documents/Burndown.xlsx'], destination='The/quick/brown/fox') @@ -153,11 +150,9 @@ Create Public Link - NA: No Access """ - """When logged in as a Team Portal Administrator End User""" """Create a Read Only public link to a file that expires in 30 days""" user.files.public_link('My Files/Documents/Sample.docx') - """When logged in as a Team Portal Administrator End User""" """Create a Read Write public link to a folder that expires in 45 days""" user.files.public_link('My Files/Documents/Sample.docx', 'RW', 45) @@ -170,13 +165,9 @@ Get Permalink .. code:: python - """When logged in as a Team Portal Administrator End User""" - """Create permalink to a file""" - user.files.permalink('My Files/Documents/Sample.docx') + user.files.permalink('My Files/Documents/Sample.docx') # file - """When logged in as a Team Portal Administrator End User""" - """Create permalink to a folder""" - user.files.permalink('My Files/Documents') + user.files.permalink('My Files/Documents') # folder Create Directories @@ -187,10 +178,8 @@ Create Directories .. code:: python - """When logged in as a Global Administrator""" admin.files.mkdir('Users/John Smith/My Files/Documents') - """When logged in as a Team Portal Administrator End User""" user.files.mkdir('My Files/Documents') .. automethod:: cterasdk.core.files.browser.CloudDrive.makedirs @@ -198,10 +187,8 @@ Create Directories .. code:: python - """When logged in as a Global Administrator""" admin.files.makedirs('Users/John Smith/My Files/The/quick/brown/fox') - """When logged in as a Team Portal Administrator End User""" user.files.makedirs('The/quick/brown/fox') Rename @@ -212,10 +199,8 @@ Rename .. code:: python - """When logged in as a Global Administrator""" admin.files.rename('Users/John Smith/My Files/Documents/Sample.docx', 'Wizard Of Oz.docx') - """When logged in as a tenant user or admin""" user.files.makedirs('My Files/Documents/Sample.docx', 'Wizard Of Oz.docx') Delete @@ -226,10 +211,8 @@ Delete .. code:: python - """When logged in as a Global Administrator""" admin.files.delete(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx']) - """When logged in as a Team Portal Administrator End User""" user.files.delete(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx']) Undelete @@ -240,10 +223,8 @@ Undelete .. code:: python - """When logged in as a Global Administrator""" admin.files.undelete(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx']) - """When logged in as a Team Portal Administrator End User""" user.files.undelete(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx']) Move @@ -252,12 +233,12 @@ Move .. automethod:: cterasdk.core.files.browser.CloudDrive.move :noindex: + To resolve file conflicts, use :py:class:`cterasdk.core.types.ConflictResolver` + .. code:: python - """When logged in as a Global Administrator""" admin.files.move(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx'], destination='Users/John Smith/The/quick/brown/fox') - """When logged in as a Team Portal Administrator End User""" user.files.move(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx'], destination='The/quick/brown/fox') Upload @@ -267,10 +248,8 @@ Upload .. code:: python - """When logged in as a Global Administrator""" admin.files.upload(r'C:\Users\admin\Downloads\Tree.jpg', 'Users/John Smith/My Files/Images') - """When logged in as a Team Portal Administrator End User""" user.files.upload(r'C:\Users\admin\Downloads\Tree.jpg', 'My Files/Images') @@ -282,7 +261,6 @@ Collaboration Shares .. code:: python - """When logged in as a Team Portal Administrator End User""" """ Share with a local user and a local group. @@ -302,7 +280,6 @@ Collaboration Shares .. code:: python - """When logged in as a Team Portal Administrator End User""" """ Share with an external recipient @@ -322,7 +299,6 @@ Collaboration Shares .. code:: python - """When logged in as a Team Portal Administrator End User""" """ Share with a domain groups @@ -342,7 +318,6 @@ Collaboration Shares .. code:: python - """When logged in as a Team Portal Administrator End User""" """ Add collaboration shares members. @@ -360,7 +335,6 @@ Collaboration Shares .. code:: python - """When logged in as a Team Portal Administrator End User""" """Remove 'Alice' and 'Engineering' from the List of Recipients""" alice = core_types.UserAccount('alice') diff --git a/tests/ut/aio/test_notifications.py b/tests/ut/aio/test_notifications.py index d3d509a7..ffec094e 100644 --- a/tests/ut/aio/test_notifications.py +++ b/tests/ut/aio/test_notifications.py @@ -108,9 +108,10 @@ async def test_ancestors_error(self): )) ) )) - with self.assertRaises(exceptions.transport.HTTPError) as error: + with self.assertRaises(exceptions.notifications.AncestorsError) as error: await notifications.Notifications(self._global_admin).ancestors(self._descendant) - self.assertEqual(error.exception.error.request.url, url) + self.assertEqual(str(error.exception), + f'Could not retrieve ancestors for: {self._descendant.folder_id}:{self._descendant.guid}') @staticmethod def _create_parameter(folder_ids=None, cursor=None, max_results=None, timeout=None): diff --git a/tests/ut/core/admin/test_copy.py b/tests/ut/core/admin/test_copy.py index 0598fccd..cf55f81d 100644 --- a/tests/ut/core/admin/test_copy.py +++ b/tests/ut/core/admin/test_copy.py @@ -16,8 +16,8 @@ def test_copy_no_wait(self): src = 'cloud/Users' dst = 'public' self._init_global_admin(execute_response=expected_response) - actual_response = io.copy(self._global_admin, self._get_object_path(src), destination=self._get_object_path(dst), wait=False) - self.assertEqual(expected_response, actual_response.ref) + actual_response = io.copy(self._global_admin, self._get_object_path(src), destination=self._get_object_path(dst)) + self.assertEqual(expected_response, actual_response) self._global_admin.api.execute.assert_called_once_with('', 'copyResources', mock.ANY) expected_copy_param = self._get_expected_copy_params(src, dst) actual_copy_param = self._global_admin.api.execute.call_args[0][2] @@ -26,6 +26,7 @@ def test_copy_no_wait(self): def _get_expected_copy_params(self, src, dst): o = Object() o._classname = 'ActionResourcesParam' # pylint: disable=protected-access + o.startFrom = None src_dst_obj = Object() src_dst_obj._classname = 'SrcDstParam' # pylint: disable=protected-access src_path = self._get_object_path(src) diff --git a/tests/ut/core/user/base_user.py b/tests/ut/core/user/base_user.py index 953b95e1..43f36940 100644 --- a/tests/ut/core/user/base_user.py +++ b/tests/ut/core/user/base_user.py @@ -26,6 +26,7 @@ def _create_action_resource_param(self, sources, destinations=None): action_resource_param = Object() action_resource_param._classname = 'ActionResourcesParam' # pylint: disable=protected-access action_resource_param.urls = [] + action_resource_param.startFrom = None for idx, source in enumerate(sources): param = Object() param._classname = 'SrcDstParam' # pylint: disable=protected-access diff --git a/tests/ut/core/user/test_browser.py b/tests/ut/core/user/test_browser.py index 4c0cb6e8..d094446b 100644 --- a/tests/ut/core/user/test_browser.py +++ b/tests/ut/core/user/test_browser.py @@ -11,6 +11,7 @@ class TestCoreFilesBrowser(base_admin.BaseCoreTest): def setUp(self): super().setUp() self.files = CloudDrive(self._global_admin) + self._await_or_future_mock = self.patch_call('cterasdk.core.files.browser.await_or_future') def test_versions(self): path = 'cloud/Users' @@ -57,49 +58,54 @@ def test_rename(self): new_name = 'Names' rename_mock = self.patch_call('cterasdk.core.files.io.rename') self.files.rename(path, new_name) - rename_mock.assert_called_once_with(self._global_admin, mock.ANY, new_name, wait=True) + rename_mock.assert_called_once_with(self._global_admin, mock.ANY, new_name) actual_ctera_path = rename_mock.call_args[0][1] self.assertEqual(actual_ctera_path.absolute, TestCoreFilesBrowser._create_expected_path(TestCoreFilesBrowser._base_path, path)) + self._await_or_future_mock.assert_called_once_with(self._global_admin, mock.ANY, True) def test_delete(self): path = 'cloud/Users' rm_mock = self.patch_call('cterasdk.core.files.io.remove') self.files.delete(path) - rm_mock.assert_called_once_with(self._global_admin, mock.ANY, wait=True) + rm_mock.assert_called_once_with(self._global_admin, mock.ANY) actual_ctera_path = rm_mock.call_args[0][1] self.assertEqual(actual_ctera_path.absolute, TestCoreFilesBrowser._create_expected_path(TestCoreFilesBrowser._base_path, path)) + self._await_or_future_mock.assert_called_once_with(self._global_admin, mock.ANY, True) def test_undelete(self): path = 'cloud/Users' recover_mock = self.patch_call('cterasdk.core.files.io.recover') self.files.undelete(path) - recover_mock.assert_called_once_with(self._global_admin, mock.ANY, wait=True) + recover_mock.assert_called_once_with(self._global_admin, mock.ANY) actual_ctera_path = recover_mock.call_args[0][1] self.assertEqual(actual_ctera_path.absolute, TestCoreFilesBrowser._create_expected_path(TestCoreFilesBrowser._base_path, path)) + self._await_or_future_mock.assert_called_once_with(self._global_admin, mock.ANY, True) def test_move(self): src = 'cloud/Users' dst = 'public' mv_mock = self.patch_call('cterasdk.core.files.io.move') self.files.move(src, destination=dst) - mv_mock.assert_called_once_with(self._global_admin, mock.ANY, destination=mock.ANY, wait=True) + mv_mock.assert_called_once_with(self._global_admin, mock.ANY, destination=mock.ANY, resolver=None, cursor=None) actual_ctera_paths = mv_mock.call_args[0][1:] self.assertListEqual( [actual_ctera_path.absolute for actual_ctera_path in actual_ctera_paths], [TestCoreFilesBrowser._create_expected_path(TestCoreFilesBrowser._base_path, path) for path in [src]] ) + self._await_or_future_mock.assert_called_once_with(self._global_admin, mock.ANY, True) def test_copy(self): src = 'cloud/Users' dst = 'public' cp_mock = self.patch_call('cterasdk.core.files.io.copy') self.files.copy(src, destination=dst) - cp_mock.assert_called_once_with(self._global_admin, mock.ANY, destination=mock.ANY, wait=True) + cp_mock.assert_called_once_with(self._global_admin, mock.ANY, destination=mock.ANY, resolver=None, cursor=None) actual_ctera_paths = cp_mock.call_args[0][1:] self.assertListEqual( [actual_ctera_path.absolute for actual_ctera_path in actual_ctera_paths], [TestCoreFilesBrowser._create_expected_path(TestCoreFilesBrowser._base_path, path) for path in [src]] ) + self._await_or_future_mock.assert_called_once_with(self._global_admin, mock.ANY, True) def test_create_public_link_default_values(self): for access in [None, 'RW', 'RO']: diff --git a/tests/ut/edge/test_sync.py b/tests/ut/edge/test_sync.py index 1efd0c11..1bbc33c3 100644 --- a/tests/ut/edge/test_sync.py +++ b/tests/ut/edge/test_sync.py @@ -89,7 +89,10 @@ def test_refresh_cloud_drive_folders(self): def test_evict_wait(self): execute_response = '/proc/bgtasks/6192' - get_response = munch.Munch(dict(id=1, name='task', status='completed', startTime='start', endTime='end')) + get_response = munch.Munch(dict(id=1, name='task', status='completed', + startTime='2025-08-25T00:04:59.601200', + elapsedTime=5, percentage=1, description='background task', + endTime='2025-08-25T00:05:59.601200', result=None)) self._init_filer(get_response=get_response, execute_response=execute_response) ret = sync.Sync(self._filer).evict(self._path, wait=True) self._filer.api.execute.assert_called_once_with('/config/cloudsync', 'evictFolder', mock.ANY) @@ -97,7 +100,7 @@ def test_evict_wait(self): actual_param = self._filer.api.execute.call_args[0][2] expected_param = munch.Munch(dict(_classname='evictFolderParam', path=self._path)) self._assert_equal_objects(actual_param, expected_param) - self.assertEqual(ret, get_response) + self.assertEqual(ret.description, get_response.description) def test_evict_no_wait(self): execute_response = '/proc/bgtasks/6192'