diff --git a/cterasdk/asynchronous/core/files/browser.py b/cterasdk/asynchronous/core/files/browser.py index 9ef305bb..273518f3 100644 --- a/cterasdk/asynchronous/core/files/browser.py +++ b/cterasdk/asynchronous/core/files/browser.py @@ -1,5 +1,5 @@ from ....cio.core import CorePath -from ....lib import FileSystem +from ....lib.storage import asynfs, commonfs from ..base_command import BaseCommand from . import io @@ -9,7 +9,6 @@ class FileBrowser(BaseCommand): def __init__(self, core): super().__init__(core) self._scope = f'/{self._core.context}/webdav' - self._filesystem = FileSystem.instance() async def handle(self, path): """ @@ -30,6 +29,38 @@ async def handle_many(self, directory, *objects): handle_many_function = await io.handle_many(self.normalize(directory), *objects) return await handle_many_function(self._core) + async def download(self, path, destination=None): + """ + Download a file + + :param str path: Path + :param str,optional destination: + File destination, if it is a directory, the original filename will be kept, defaults to the default directory + """ + directory, name = commonfs.determine_directory_and_filename(path, destination=destination) + handle = await self.handle(path) + return await asynfs.write(directory, name, handle) + + async def download_many(self, target, objects, destination=None): + """ + Download selected files and/or directories as a ZIP archive. + + .. warning:: + The provided list of objects is not validated. Only existing files and directories + will be included in the resulting ZIP file. + + :param str target: + Path to the cloud folder containing the files and directories to download. + :param list[str] objects: + List of file and/or directory names to include in the download. + :param str destination: + Optional. Path to the destination file or directory. If a directory is provided, + the original filename will be preserved. Defaults to the default download directory. + """ + directory, name = commonfs.determine_directory_and_filename(target, objects, destination=destination, archive=True) + handle = await self.handle_many(target, *objects) + return await asynfs.write(directory, name, handle) + async def listdir(self, path, depth=None, include_deleted=False): """ List Directory @@ -116,7 +147,7 @@ async def upload_file(self, path, destination): :param str destination: Remote path """ with open(path, 'rb') as handle: - metadata = self._filesystem.properties(path) + metadata = commonfs.properties(path) response = await self.upload(metadata['name'], metadata['size'], destination, handle) return response diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index 0704c1a8..6e09c265 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -3,7 +3,7 @@ import cterasdk.settings from ...cio.core import CorePath from ...exceptions import CTERAException -from ...lib import FileSystem +from ...lib.storage import synfs, commonfs from ..base_command import BaseCommand from . import io @@ -13,7 +13,6 @@ class FileBrowser(BaseCommand): def __init__(self, core): super().__init__(core) self._scope = f'/{self._core.context}/webdav' - self._filesystem = FileSystem.instance() def handle(self, path): """ @@ -42,24 +41,29 @@ def download(self, path, destination=None): :param str,optional destination: File destination, if it is a directory, the original filename will be kept, defaults to the default directory """ - directory, name = self.determine_directory_and_filename(path, destination=destination) + directory, name = commonfs.determine_directory_and_filename(path, destination=destination) handle = self.handle(path) - return self._filesystem.save(directory, name, handle) + return synfs.write(directory, name, handle) - def download_as_zip(self, target, objects, destination=None): + def download_many(self, target, objects, destination=None): """ - Download a list of files and/or directories from a cloud folder as a ZIP file + Download selected files and/or directories as a ZIP archive. - .. warning:: The list of files is not validated. The ZIP file will include only the existing files and directories + .. warning:: + The provided list of objects is not validated. Only existing files and directories + will be included in the resulting ZIP file. - :param str target: Path to the cloud directory - :param list[str] objects: List of files and/or directories in the cloud folder to download - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str target: + Path to the cloud folder containing the files and directories to download. + :param list[str] objects: + List of file and/or directory names to include in the download. + :param str destination: + Optional. Path to the destination file or directory. If a directory is provided, + the original filename will be preserved. Defaults to the default download directory. """ - directory, name = self.determine_directory_and_filename(target, objects, destination=destination, archive=True) + directory, name = commonfs.determine_directory_and_filename(target, objects, destination=destination, archive=True) handle = self.handle_many(target, *objects) - return self._filesystem.save(directory, name, handle) + return synfs.write(directory, name, handle) def listdir(self, path, depth=None, include_deleted=False): """ @@ -120,31 +124,6 @@ def permalink(self, path): return contents[0].permalink raise FileNotFoundError('File not found.', path) - def determine_directory_and_filename(self, p, objects=None, destination=None, archive=False): - """ - Determine location to save file. - - :param str p: Path. - :param list[str],optional objects: List of files or folders - :param str,optional destination: Destination - :param bool,optional archive: Compressed archive - :returns: Directory and file name - :rtype: tuple[str] - """ - directory, name = None, None - if destination: - directory, name = self._filesystem.split_file_directory(destination) - else: - directory = self._filesystem.downloads_directory() - - if not name: - normalized = self.normalize(p) - if archive: - name = self._filesystem.compute_zip_file_name(normalized.absolute, objects) - else: - name = normalized.name - return directory, name - def normalize(self, entries): return CorePath.instance(self._scope, entries) @@ -171,7 +150,7 @@ def upload_file(self, path, destination): :param str destination: Remote path """ with open(path, 'rb') as handle: - metadata = self._filesystem.properties(path) + metadata = commonfs.properties(path) response = self.upload(metadata['name'], metadata['size'], destination, handle) return response diff --git a/cterasdk/core/ssl.py b/cterasdk/core/ssl.py index 782f875b..35e86ff6 100644 --- a/cterasdk/core/ssl.py +++ b/cterasdk/core/ssl.py @@ -2,7 +2,8 @@ from zipfile import ZipFile from .base_command import BaseCommand -from ..lib import FileSystem, X509Certificate, PrivateKey, TempfileServices, create_certificate_chain +from ..lib import X509Certificate, PrivateKey, TempfileServices, create_certificate_chain +from ..lib.storage import commonfs, synfs class SSL(BaseCommand): @@ -10,10 +11,6 @@ class SSL(BaseCommand): Portal SSL Certificate APIs """ - def __init__(self, portal): - super().__init__(portal) - self._filesystem = FileSystem.instance() - def get(self): """ Retrieve details of the current installed SSL certificate @@ -39,14 +36,15 @@ def export(self, destination=None): :param str,optional destination: File destination, defaults to the default directory """ - directory, filename = self._filesystem.generate_file_location(destination, 'certificate.zip') + directory, filename = commonfs.generate_file_destination(destination, 'certificate.zip') logging.getLogger('cterasdk.core').info('Exporting SSL certificate.') handle = self._core.ctera.handle('/preview/exportCertificate') - filepath = self._filesystem.save(directory, filename, handle) + filepath = synfs.write(directory, filename, handle) logging.getLogger('cterasdk.core').info('Exported SSL certificate. %s', {'filepath': filepath}) return filepath - def create_zip_archive(self, private_key, *certificates): + @staticmethod + def create_zip_archive(private_key, *certificates): """ Create a ZIP archive that can be imported to CTERA Portal @@ -57,8 +55,8 @@ def create_zip_archive(self, private_key, *certificates): key_basename = 'private.key' key_object = PrivateKey.load_private_key(private_key) - key_filepath = FileSystem.join(tempdir, key_basename) - self._filesystem.write(key_filepath, key_object.pem_data) + key_filepath = commonfs.join(tempdir, key_basename) + synfs.overwrite(key_filepath, key_object.pem_data) cert_basename = 'certificate' certificates = [X509Certificate.load_certificate(certificate) for certificate in certificates] @@ -66,13 +64,13 @@ def create_zip_archive(self, private_key, *certificates): certificate_chain_zip_archive = None if certificate_chain: - certificate_chain_zip_archive = FileSystem.join(tempdir, f'{cert_basename}.zip') + certificate_chain_zip_archive = commonfs.join(tempdir, f'{cert_basename}.zip') with ZipFile(certificate_chain_zip_archive, 'w') as zip_archive: zip_archive.write(key_filepath, key_basename) for idx, certificate in enumerate(certificate_chain): filename = f'{cert_basename}{idx if idx > 0 else ""}.crt' - filepath = FileSystem.join(tempdir, filename) - self._filesystem.write(filepath, certificate.pem_data) + filepath = commonfs.join(tempdir, filename) + synfs.overwrite(filepath, certificate.pem_data) zip_archive.write(filepath, filename) return certificate_chain_zip_archive @@ -92,11 +90,11 @@ def import_from_chain(self, private_key, *certificates): :param str private_key: The PEM-encoded private key, or a path to the PEM-encoded private key file :param list[str] certificates: The PEM-encoded certificates, or a list of paths of the PEM-encoded certificate files """ - zipflie = self.create_zip_archive(private_key, *certificates) + zipflie = SSL.create_zip_archive(private_key, *certificates) return self.import_from_zip(zipflie) def _import_certificate(self, zipfile): - self._filesystem.properties(zipfile) + commonfs.properties(zipfile) logging.getLogger('cterasdk.core').info('Uploading SSL certificate.') with open(zipfile, 'rb') as fd: response = self._core.api.form_data( diff --git a/cterasdk/edge/config.py b/cterasdk/edge/config.py index 4018509a..ab4bc249 100644 --- a/cterasdk/edge/config.py +++ b/cterasdk/edge/config.py @@ -5,17 +5,17 @@ from ..exceptions import CTERAException from ..convert import fromxmlstr, toxmlstr from ..common import Device, delete_attrs -from ..lib import FileSystem, TempfileServices +from ..lib import TempfileServices +from ..lib.storage import commonfs, synfs from .base_command import BaseCommand +logger = logging.getLogger('cterasdk.edge') + + class Config(BaseCommand): """ Edge Filer General Configuration APIs """ - def __init__(self, edge): - super().__init__(edge) - self._filesystem = FileSystem.instance() - def get_location(self): """ Get the location of the Edge Filer @@ -31,7 +31,7 @@ def set_location(self, location): :param str location: New location to set :return str: The new location """ - logging.getLogger('cterasdk.edge').info('Configuring device location. %s', {'location': location}) + logger.info('Configuring device location. %s', {'location': location}) return self._edge.api.put('/config/device/location', location) def get_hostname(self): @@ -49,7 +49,7 @@ def set_hostname(self, hostname): :param str hostname: New hostname to set :return str: The new hostname """ - logging.getLogger('cterasdk.edge').info('Configuring device hostname. %s', {'hostname': hostname}) + logger.info('Configuring device hostname. %s', {'hostname': hostname}) return self._edge.api.put('/config/device/hostname', hostname) def import_config(self, config, exclude=None): @@ -64,19 +64,19 @@ def import_config(self, config, exclude=None): if isinstance(config, Device): database = copy.deepcopy(config) elif isinstance(config, str): - database = self.load_config(config) + database = Config.load_config(config) if exclude: delete_attrs(database, exclude) - path = self._filesystem.join(TempfileServices.mkdir(), f'{self._edge.session().address}.xml') - self._filesystem.write(path, toxmlstr(database, True).encode('utf-8')) + path = commonfs.join(TempfileServices.mkdir(), f'{self._edge.session().address}.xml') + synfs.overwrite(path, toxmlstr(database, True).encode('utf-8')) return self._import_configuration(path) def _import_configuration(self, path): - self._filesystem.properties(path) - logging.getLogger('cterasdk.edge').info('Importing Edge Filer configuration.') + commonfs.properties(path) + logger.info('Importing Edge Filer configuration.') with open(path, 'rb') as fd: response = self._edge.api.form_data( '/config', @@ -86,18 +86,19 @@ def _import_configuration(self, path): config=fd ) ) - logging.getLogger('cterasdk.edge').info('Imported Edge Filer configuration.') + logger.info('Imported Edge Filer configuration.') return response - def load_config(self, config): + @staticmethod + def load_config(config): """ Load the Edge Filer configuration :param str config: A string or a path to the Edge Filer configuration file """ data = None - if self._filesystem.exists(config): - logging.getLogger('cterasdk.edge').info('Reading the Edge Filer configuration from file. %s', {'path': config}) + if commonfs.exists(config): + logger.info('Reading the Edge Filer configuration from file. %s', {'path': config}) with open(config, 'r', encoding='utf-8') as f: data = f.read() else: @@ -105,9 +106,9 @@ def load_config(self, config): database = fromxmlstr(data) if database: - logging.getLogger('cterasdk.edge').info('Completed parsing the Edge Filer configuration. %s', {'firmware': database.firmware}) + logger.info('Completed parsing the Edge Filer configuration. %s', {'firmware': database.firmware}) return database - logging.getLogger('cterasdk.edge').error("Failed parsing the Edge Filer's configuration.") + logger.error("Failed parsing the Edge Filer's configuration.") raise CTERAException("Failed parsing the Edge Filer's configuration") def export(self, destination=None): @@ -118,11 +119,12 @@ def export(self, destination=None): File destination, defaults to the default directory """ default_filename = self._edge.host() + datetime.now().strftime('_%Y-%m-%dT%H_%M_%S') + '.xml' - directory, filename = self._filesystem.generate_file_location(destination, default_filename) - logging.getLogger('cterasdk.edge').info('Exporting configuration. %s', {'host': self._edge.host()}) + directory, filename = commonfs.generate_file_destination(destination, default_filename) + logger.info('Exporting configuration. %s', {'host': self._edge.host()}) handle = self._edge.api.handle('/export') - filepath = FileSystem.instance().save(directory, filename, handle) - logging.getLogger('cterasdk.edge').info('Exported configuration. %s', {'filepath': filepath}) + filepath = synfs.write(directory, filename, handle) + logger.info('Exported configuration. %s', {'filepath': filepath}) + return filepath def is_wizard_enabled(self): """ @@ -145,5 +147,5 @@ def disable_wizard(self): return self._set_wizard(False) def _set_wizard(self, state): - logging.getLogger('cterasdk.edge').info('Disabling first time wizard') + logger.info('Disabling first time wizard') return self._edge.api.put('/config/gui/openFirstTimeWizard', state) diff --git a/cterasdk/edge/files/browser.py b/cterasdk/edge/files/browser.py index 7e91e8f3..4fdb05ca 100644 --- a/cterasdk/edge/files/browser.py +++ b/cterasdk/edge/files/browser.py @@ -1,16 +1,12 @@ from ..base_command import BaseCommand from ...cio.edge import EdgePath -from ...lib import FileSystem +from ...lib.storage import synfs, commonfs from . import io class FileBrowser(BaseCommand): """ Edge Filer File Browser APIs """ - def __init__(self, edge): - super().__init__(edge) - self._filesystem = FileSystem.instance() - def listdir(self, path): """ List Directory @@ -46,24 +42,29 @@ def download(self, path, destination=None): :param str,optional destination: File destination, if it is a directory, the original filename will be kept, defaults to the default directory """ - directory, name = self.determine_directory_and_filename(path, destination=destination) + directory, name = commonfs.determine_directory_and_filename(path, destination=destination) handle = self.handle(path) - return self._filesystem.save(directory, name, handle) + return synfs.write(directory, name, handle) - def download_as_zip(self, target, objects, destination=None): + def download_many(self, target, objects, destination=None): """ - Download a list of files and/or directories from a cloud folder as a ZIP file + Download selected files and/or directories as a ZIP archive. - .. warning:: The list of files is not validated. The ZIP file will include only the existing files and directories + .. warning:: + The provided list of objects is not validated. Only existing files and directories + will be included in the resulting ZIP file. - :param str target: Path to a directory - :param list[str] objects: List of files and/or directories in the cloud folder to download - :param str,optional destination: - File destination, if it is a directory, the filename will be calculated, defaults to the default directory + :param str target: + Path to the cloud folder containing the files and directories to download. + :param list[str] objects: + List of file and/or directory names to include in the download. + :param str destination: + Optional. Path to the destination file or directory. If a directory is provided, + the original filename will be preserved. Defaults to the default download directory. """ - directory, name = self.determine_directory_and_filename(target, objects, destination=destination, archive=True) + directory, name = commonfs.determine_directory_and_filename(target, objects, destination=destination, archive=True) handle = self.handle_many(target, *objects) - return self._filesystem.save(directory, name, handle) + return synfs.write(directory, name, handle) def upload(self, name, destination, handle): """ @@ -83,7 +84,7 @@ def upload_file(self, path, destination): :param str path: Local path :param str destination: Remote path """ - metadata = self._filesystem.properties(path) + metadata = commonfs.properties(path) with open(path, 'rb') as handle: response = self.upload(metadata['name'], destination, handle) return response @@ -136,31 +137,6 @@ def delete(self, path): """ return io.remove(self._edge, self.normalize(path)) - def determine_directory_and_filename(self, p, objects=None, destination=None, archive=False): - """ - Determine location to save file. - - :param str p: Path. - :param list[str],optional objects: List of files or folders - :param str,optional destination: Destination - :param bool,optional archive: Compressed archive - :returns: Directory and file name - :rtype: tuple[str] - """ - directory, name = None, None - if destination: - directory, name = self._filesystem.split_file_directory(destination) - else: - directory = self._filesystem.downloads_directory() - - if not name: - normalized = self.normalize(p) - if archive: - name = self._filesystem.compute_zip_file_name(normalized.absolute, objects) - else: - name = normalized.name - return directory, name - @staticmethod def normalize(path): return EdgePath('/', path) diff --git a/cterasdk/edge/firmware.py b/cterasdk/edge/firmware.py index 6143f25f..1eceab1b 100644 --- a/cterasdk/edge/firmware.py +++ b/cterasdk/edge/firmware.py @@ -1,4 +1,4 @@ -from ..lib import FileSystem +from ..lib.storage import commonfs from ..exceptions import CTERAException from .base_command import BaseCommand @@ -12,10 +12,6 @@ class UploadTaskStatus(): class Firmware(BaseCommand): """ Edge Filer Firmware upgrade API """ - def __init__(self, edge): - super().__init__(edge) - self._filesystem = FileSystem.instance() - def upgrade(self, file_path, reboot=True, wait_for_reboot=True): """ Upgrade the Filer firmware with the provided file @@ -32,7 +28,7 @@ def upgrade(self, file_path, reboot=True, wait_for_reboot=True): self._edge.power.reboot(wait=wait_for_reboot) def _upload_firmware(self, file_path): - self._filesystem.properties(file_path) + commonfs.properties(file_path) with open(file_path, 'rb') as fd: return self._edge.api.form_data( 'proc/firmware', diff --git a/cterasdk/lib/storage/__init__.py b/cterasdk/lib/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cterasdk/lib/storage/asynfs.py b/cterasdk/lib/storage/asynfs.py new file mode 100644 index 00000000..e53f22b3 --- /dev/null +++ b/cterasdk/lib/storage/asynfs.py @@ -0,0 +1,39 @@ +import logging +import aiofiles +from .commonfs import write_new_version, ResultContext + + +logger = logging.getLogger('cterasdk.filesystem') + + +async def write(directory, name, handle): + """ + Write bytes to disk. + + :param str directory: Directory + :param str name: Name + :param bytes handle: Handle + :returns: Path + :rtype: str + """ + ctx = ResultContext() + with write_new_version(directory, name) as tempfile: + await overwrite(tempfile, handle) + return ctx.value + + +async def overwrite(p, handle): + """ + Write, without validation. + + :param Path p: Path + :param bytes handle: Handle + """ + async with aiofiles.open(p, 'w+b') as fd: + if isinstance(fd, bytes): + await fd.write(handle) + else: + async for chunk in handle.async_iter_content(chunk_size=8192): + await fd.write(chunk) + logger.debug('Wrote: %s', p.as_posix()) + return p.as_posix() diff --git a/cterasdk/lib/storage/commonfs.py b/cterasdk/lib/storage/commonfs.py new file mode 100644 index 00000000..83c8a226 --- /dev/null +++ b/cterasdk/lib/storage/commonfs.py @@ -0,0 +1,266 @@ +import os +import errno +import logging +import mimetypes +from pathlib import Path +from contextlib import contextmanager +import cterasdk.settings + + +logger = logging.getLogger('cterasdk.filesystem') + + +def exists(path): + """ + Check if a file or a directory exists + + :param str path: Path + :returns: ``True`` if exists, ``False`` otherwise. + :rtype: bool + """ + return Path(path).exists() + + +def expanduser(path): + """ + Return a new path with expanded ~ and ~user constructs + + :param str path: Path + :returns: Absolute Path. + :rtype: Path object + """ + return Path(path).expanduser() + + +def is_dir(path): + """ + Check is a directory. + + :param str path: Path + :returns: ``True`` if a directory, ``False`` otherwise. + :rtype: bool + """ + p = expanduser(path) + return p.is_dir() + + +def join(*paths): + """ + Join Path objects. + + :param list[Path] paths: Path objects + :rtype: Path + """ + return Path(*paths) + + +def properties(path): + """ + File properties. + + :param str path: Path + :returns: File name, size and type + :rtype: dict + :raises: FileNotFoundError + """ + p = expanduser(path) + + if not p.exists() or not p.is_file(): + logger.error('File not found: %s', p.as_posix()) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), p.as_posix()) + + return dict( + name=p.name, + size=str(p.stat().st_size), + mimetype=mimetypes.guess_type(path) + ) + + +def downloads(): + """ + Get downloads directory. + + :returns: Directory path + :rtype: Path object + """ + directory = expanduser(cterasdk.settings.downloads.location) + if not is_dir(directory): + logger.error('Directory not found: %s', directory) + raise FileNotFoundError(errno.ENOENT, 'Directory not found', directory) + return directory + + +def determine_zip_archive_name(directory, objects): + """ + Name the zip archive after the folder name, unless the directory contains only one object. + + :rtype: str + """ + if len(objects) > 1: + path = Path(directory) + else: + path = Path(objects[0]) + return f'{path.stem}.zip' + + +def new_version(filename, version): + """ + Append version number to file name. + + :param str filename: File name + :param int version: File version + :returns: File name appended with a version number + :rtype: str + """ + idx = filename.rfind('.') + extension = '' + if idx > 0: + name = filename[:idx] + extension = filename[idx:] + else: + name = filename + return f'{name} ({str(version)}){extension}' + + +def rename(source, new_name): + """ + Rename a file or a directory. + + :param str source: Path + :param str new_name: New name + :returns: Path after rename + :rtype: str + """ + source = expanduser(source) + if not source.exists(): + logger.error('File not found: %s', source.as_posix()) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), source.as_posix()) + destination = source.parent.joinpath(new_name) + source.rename(destination) + return destination.as_posix() + + +class ResultContext: + """Context Manager Result Context""" + + def __init__(self): + self._value = None + + @property + def value(self): + return self._value + + @value.setter + def value(self, v): + self._value = v + + +@contextmanager +def write_new_version(directory, name, *, ctx=None): + """ + Context manager for writing a file without conflicts. + + :param str directory: Parent directory + :param str name: File name + :param ResultContext,optional ctx: Result Context + :returns: Path + :rtype: str + :raises: FileNotFoundError + """ + parent = expanduser(directory) + if not parent.exists() or not is_dir(parent): + logger.error('Directory not found: %s', parent.as_posix()) + raise FileNotFoundError(errno.ENOENT, 'Directory not found', directory) + + tempfile = parent.joinpath(f'{name}.ctera') + yield tempfile + + origin, version = name, 0 + while True: + try: + rename(tempfile, name) + break + except (FileExistsError, IsADirectoryError): + logger.debug('File exists: %s', parent.joinpath(name)) + version = version + 1 + name = new_version(origin, version) + + p = parent.joinpath(name) + logger.info('Saved: %s', p.as_posix()) + + if ctx and isinstance(ctx, ResultContext): + ctx.value = p.as_posix() + + +def split_file_directory(location): + """ + Split file and directory. + + :param str path: Path + + Returns: + 1. (parent directory, file name), if a file exists + 2. (parent directory, file name), if a directory exists + 3. (parent directory, file name), if the parent directory exists + 4. Raises ``FileNotFoundError`` if neither the object nor the parent directory exist + """ + p = expanduser(location) + if p.exists(): + if p.is_dir(): + filename = None + else: + filename = p.name + p = p.parent + elif p.parent.exists(): + filename = p.name + p = p.parent + else: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), location) + return str(p.resolve()), filename + + +def generate_file_destination(destination=None, default_name=None): + """ + Compute destination file path. + + :param str location: Path to a file or a folder + :param str default_name: Default file name, unless ``location`` already specifies a file path + :returns: Tuple including the destination directory and file name + :rtype: tuple(str, str) + """ + parent = name = None + if destination: + parent, name = split_file_directory(destination) + else: + parent = downloads() + + if not name: + name = default_name + + return (parent, name) + + +def determine_directory_and_filename(p, objects=None, destination=None, archive=False): + """ + Determine location to save file. + + :param str p: Path. + :param list[str],optional objects: List of files or folders + :param str,optional destination: Destination + :param bool,optional archive: Compressed archive + :returns: Directory and file name + :rtype: tuple[str] + """ + directory, name = None, None + if destination: + directory, name = split_file_directory(destination) + else: + directory = downloads() + + if not name: + normalized = Path(p) + if archive: + name = determine_zip_archive_name(p, objects) + else: + name = normalized.name + return directory, name diff --git a/cterasdk/lib/storage/synfs.py b/cterasdk/lib/storage/synfs.py new file mode 100644 index 00000000..38814755 --- /dev/null +++ b/cterasdk/lib/storage/synfs.py @@ -0,0 +1,38 @@ +import logging +from .commonfs import write_new_version, ResultContext + + +logger = logging.getLogger('cterasdk.filesystem') + + +def write(directory, name, handle): + """ + Write bytes to disk. + + :param str directory: Directory + :param str name: Name + :param bytes handle: Handle + :returns: Path + :rtype: str + """ + ctx = ResultContext() + with write_new_version(directory, name, ctx=ctx) as tempfile: + overwrite(tempfile, handle) + return ctx.value + + +def overwrite(p, handle): + """ + Write, without validation. + + :param Path p: Path + :param bytes handle: Handle + """ + with open(p, 'w+b') as fd: + if isinstance(handle, bytes): + fd.write(handle) + else: + for chunk in handle.iter_content(chunk_size=8192): + fd.write(chunk) + logger.debug('Wrote: %s', p.as_posix()) + return p.as_posix() diff --git a/cterasdk/lib/tempfile.py b/cterasdk/lib/tempfile.py index bcceb9cc..34717763 100644 --- a/cterasdk/lib/tempfile.py +++ b/cterasdk/lib/tempfile.py @@ -6,6 +6,9 @@ from .registry import Registry +logger = logging.getLogger('cterasdk.filesystem') + + class TempfileServices: __tempdir_prefix = 'cterasdk-' @@ -15,18 +18,18 @@ def mkdir(): registry = Registry.instance() tempdir = registry.get('tempdir') if tempdir is None: - logging.getLogger('cterasdk.filesystem').debug('Creating temporary directory.') + logger.debug('Creating temporary directory.') tempdir = tempfile.mkdtemp(prefix=TempfileServices.__tempdir_prefix) - logging.getLogger('cterasdk.filesystem').debug('Temporary directory created. %s', {'path': tempdir}) + logger.debug('Temporary directory created. %s', {'path': tempdir}) registry.register('tempdir', tempdir) return tempdir @staticmethod def mkfile(prefix, suffix): tempdir = TempfileServices.mkdir() - logging.getLogger('cterasdk.filesystem').debug('Creating temporary file.') + logger.debug('Creating temporary file.') fd, filepath = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=tempdir) - logging.getLogger('cterasdk.filesystem').debug('Temporary file created. %s', {'path': filepath}) + logger.debug('Temporary file created. %s', {'path': filepath}) return (fd, filepath) @staticmethod @@ -35,7 +38,7 @@ def rmdir(): registry = Registry.instance() tempdir = registry.get('tempdir') if tempdir is not None: - logging.getLogger('cterasdk.filesystem').debug('Removing temporary directory. %s', {'path': tempdir}) + logger.debug('Removing temporary directory. %s', {'path': tempdir}) shutil.rmtree(path=tempdir) - logging.getLogger('cterasdk.filesystem').debug('Removed temporary directory. %s', {'path': tempdir}) + logger.debug('Removed temporary directory. %s', {'path': tempdir}) registry.remove('tempdir') diff --git a/docs/source/UserGuides/Edge/Files.rst b/docs/source/UserGuides/Edge/Files.rst index bec1343d..f68367c2 100644 --- a/docs/source/UserGuides/Edge/Files.rst +++ b/docs/source/UserGuides/Edge/Files.rst @@ -24,12 +24,12 @@ Download edge.files.download('cloud/users/Service Account/My Files/Documents/Sample.docx') -.. automethod:: cterasdk.edge.files.browser.FileBrowser.download_as_zip +.. automethod:: cterasdk.edge.files.browser.FileBrowser.download_many :noindex: .. code:: python - edge.files.download_as_zip('network-share/docs', ['Sample.docx', 'Summary.xlsx']) + edge.files.download_many('network-share/docs', ['Sample.docx', 'Summary.xlsx']) Create Directory ================ diff --git a/docs/source/UserGuides/Portal/Files.rst b/docs/source/UserGuides/Portal/Files.rst index 154596f4..03975e81 100644 --- a/docs/source/UserGuides/Portal/Files.rst +++ b/docs/source/UserGuides/Portal/Files.rst @@ -2,18 +2,36 @@ File Browser ============ +This article outlines the file-browser APIs available in the CTERA Portal, enabling programmatic access to files and directories. + +The API supports both **synchronous** and **asynchronous** implementations, allowing developers to choose the most suitable model +for their integration use case—whether real-time interactions or background processing. + + +Synchronous API +=============== + + +User Roles +---------- + +The file access APIs are available to the following user roles: + +- **Global Administrators** with the `Access End User Folders` permission enabled. +- **Team Portal Administrators** with the `Access End User Folders` permission enabled. +- **End Users**, accessing their personal cloud drive folders. + +For more information, See: `Customizing Administrator Roles `_ + List -==== +---- .. automethod:: cterasdk.core.files.browser.FileBrowser.listdir :noindex: -Use the following API to list the contents of a directory as a Global Administrator. -This API requires that the administrator has *Access End User Folders*. -For more details, see `Customizing Administrator Roles `_ - .. code:: python + """List directories as a Global Administrator""" with GlobalAdmin('tenant.ctera.com') as admin: admin.login('admin-user', 'admin-pass') @@ -26,14 +44,9 @@ For more details, see `Customizing Administrator Roles `_ - .. code:: python + """List directories as a Team Portal Administrator or End User""" with ServicesPortal('tenant.ctera.com') as user: user.login('username', 'user-password') for f in user.files.listdir('My Files/Documents'): @@ -66,7 +79,7 @@ For more details, see `Viewing Folder Content `_ @@ -401,3 +425,111 @@ The following section includes examples on how to instantiate an S3 client using client.download_file(r'./data-management-document.docx', 'my-bucket-name', 'data-management-document-copy.docx') # for more information, please refer to the Amazon SDK for Python (boto3) documentation. + + +Asynchronous API +================ + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.handle + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.handle_many + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.download + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.download_many + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.listdir + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.versions + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.walk + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.public_link + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.copy + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.move + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.FileBrowser.permalink + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.upload + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.upload_file + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.mkdir + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.makedirs + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.rename + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.delete + :noindex: + +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.undelete + :noindex: + + +.. code:: python + + """Access a Global Administrator""" + async with AsyncGlobalAdmin('global.ctera.com') as admin: + await admin.login('username', 'password') + await admin.portals.browse('corp') # access files in the 'corp' Team Portal tenant + + """Create directories recursively""" + await admin.files.makedirs('Users/John Smith/My Files/the/quick/brown/fox') + + """Create a 'Documents' directory""" + await admin.files.mkdir('Users/John Smith/Documents') + + """Walk 'John Smith's My Files directory""" + async for i in admin.files.walk('Users/John Smith/My Files'): + print(i.name, i.size, i.lastmodified, i.permalink) + + """List all files in a directory""" + documents = [i.name async for i in admin.files.listdir('Users/John Smith/Documents') if i.isfile] + + """Rename a directory""" + await admin.files.rename('Users/John Smith/Documents', 'Documents360') + + """Download""" + await admin.files.download('Users/John Smith/My Files/Sunrise.png') + await admin.files.download('Users/John Smith/My Files/Sunrise.png', 'c:/users/jsmith/downloads/Patagonia.png') + + await admin.files.download_many('Users/John Smith/Pictures', ['Sunrise.png', 'Gelato.pptx']) + await admin.files.download_many('Users/John Smith/Pictures', ['Sunrise.png', 'Gelato.pptx'], 'c:/users/jsmith/downloads/Images.zip') + + """Upload""" + await admin.files.upload_file('c:/users/jsmith/downloads/Sunset.png', '/Users/John Smith/Pictures') + + """Public Link""" + url = await admin.files.public_link('Users/John Smith/Pictures/Sunrise.png') + print(url) + +.. code:: python + + """Access a Team Portal Administrator or End User""" + async with AsyncservicesPortal('tenant.ctera.com') as user: + await user.login('username', 'password') + + """Create directories as an End User""" + await user.files.makedirs('My Files/the/quick/brown/fox') # Create a directory in your own account + + """Create directories as Team Portal Administrator""" + await user.files.makedirs('Users/John Smith/My Files/the/quick/brown/fox') # Create a directory in a user's account diff --git a/tests/ut/core/admin/test_ssl.py b/tests/ut/core/admin/test_ssl.py index e564710d..6ae8a11f 100644 --- a/tests/ut/core/admin/test_ssl.py +++ b/tests/ut/core/admin/test_ssl.py @@ -23,9 +23,9 @@ def test_thumbprint(self): def test_export_certificate(self): destination = '/home/user' filename = 'certificate.zip' - mock_split_file_directory = self.patch_call('cterasdk.core.ssl.FileSystem.generate_file_location') + mock_split_file_directory = self.patch_call('cterasdk.core.ssl.commonfs.generate_file_destination') mock_split_file_directory.return_value = (destination, filename) - mock_save = self.patch_call('cterasdk.core.ssl.FileSystem.save') + mock_save = self.patch_call('cterasdk.core.ssl.synfs.write') mock_save.return_value = f'{destination}/{filename}' handle_response = 'handle' self._init_setup(handle_response=handle_response) diff --git a/tests/ut/edge/test_config.py b/tests/ut/edge/test_config.py index 8cc068c4..c74a7271 100644 --- a/tests/ut/edge/test_config.py +++ b/tests/ut/edge/test_config.py @@ -66,9 +66,9 @@ def test_disable_first_time_wizard(self): def test_edge_config_export_default_dest(self): handle_response = 'Stream' self._init_filer(handle_response=handle_response) - mock_get_dirpath = self.patch_call("cterasdk.lib.filesystem.FileSystem.downloads_directory", + mock_get_dirpath = self.patch_call("cterasdk.lib.storage.commonfs.downloads", return_value=self._default_download_directory) - mock_save_file = self.patch_call("cterasdk.lib.filesystem.FileSystem.save") + mock_save_file = self.patch_call("cterasdk.lib.storage.synfs.write") with mock.patch.object(datetime, 'datetime', mock.Mock(wraps=datetime.datetime)) as patched: patched.now.return_value = self._current_datetime config.Config(self._filer).export() @@ -80,9 +80,9 @@ def test_edge_config_export_default_dest(self): def test_edge_config_export_target_directory_default_filename(self): handle_response = 'Stream' self._init_filer(handle_response=handle_response) - mock_get_dirpath = self.patch_call("cterasdk.lib.filesystem.FileSystem.split_file_directory", + mock_get_dirpath = self.patch_call("cterasdk.lib.storage.commonfs.split_file_directory", return_value=(self._target_directory, None)) - mock_save_file = self.patch_call("cterasdk.lib.filesystem.FileSystem.save") + mock_save_file = self.patch_call("cterasdk.lib.storage.synfs.write") with mock.patch.object(datetime, 'datetime', mock.Mock(wraps=datetime.datetime)) as patched: patched.now.return_value = self._current_datetime config.Config(self._filer).export(self._target_directory)