update to pass flake, pylint and ut

Author: Saimon Michelson

cterasdk/core/query.py

@@ -4,7 +4,7 @@
 from ..common import Object
 
 
-def database(core, path, param):
+def run(core, path, param):
     return create_callback_function(core, path, callback_response=DefaultResponse)(param)
 
 

cterasdk/core/servers.py

@@ -52,7 +52,7 @@ def system_database(self):
         :returns: Main database object.
         :rtype: cterasdk.common.object.Object
         """
-        response = query.database(self._core, '/servers', query.QueryParamBuilder().addFilter(
+        response = query.run(self._core, '/servers', query.QueryParamBuilder().addFilter(
             query.FilterBuilder('mainDB').eq(True)
         ).build())
         return response.objects[0]
@@ -118,7 +118,7 @@ class Backup(BaseCommand):
 
     def connected(self):
         """
-        Verify connectivity to the backup S3 bucket.   
+        Verify connectivity to the backup S3 bucket
         """
         return self._core.servers.system_database.backupToBucket.status == 'Connected'
 
@@ -127,7 +127,7 @@ def enable(self, bucket, interval):
         Enable Main Database Backup to an S3 Bucket
 
         :param cterasdk.core.types.Bucket bucket: Storage bucket
-        :param int interval: Backup interval in minutes        
+        :param int interval: Backup interval in minutes
         """
         database = self._core.servers.system_database
         database.backupToBucket = Object(enabled=True, exportSchedulePeriod=interval, details=bucket.database_backup_server_object())

cterasdk/direct/lib.py

@@ -1,12 +1,11 @@
 import logging
 import asyncio
-import urllib.parse
 
 from ..lib.retries import execute_with_retries
 from .types import Metadata, Block
 from .crypto import decrypt_key, decrypt_block
 from .decompressor import decompress
-from ..exceptions.transport import BadRequest, Unauthorized, Forbidden, Unprocessable, InternalServerError, HTTPError
+from ..exceptions.transport import BadRequest, Unauthorized, Unprocessable, InternalServerError, HTTPError
 from ..exceptions.direct import (
     AuthorizationError, BlockListConnectionError, BlockListTimeout, BlockValidationException, BlocksNotFoundError,
     DecompressBlockError, DecryptBlockError, DecryptKeyError, DirectIOError, DownloadConnectionError,
@@ -27,7 +26,10 @@ async def get_object(client, file_id, chunk):
     :returns: Object
     :rtype: bytes
     """
-    message = f"Downloading block (offset={chunk.offset}, length={chunk.length})"
+    message = (
+        f"Downloading block #{chunk.number} "
+        f"(offset={chunk.offset}, length={chunk.length})"
+    )
 
     if file_id:
         message += f" for file ID {file_id}"
@@ -58,8 +60,10 @@ async def get_object(client, file_id, chunk):
         "unknown": "Unknown error"
     }
 
-    message = f"Failed to download block (offset={chunk.offset}, length={chunk.length})"
-
+    message = (
+        f"Failed to download block #{chunk.number} "
+        f"(offset={chunk.offset}, length={chunk.length})"
+    )
     if file_id:
         message = message + f" for file ID {file_id}"
 
@@ -68,10 +72,6 @@ async def get_object(client, file_id, chunk):
     raise exception
 
 
-def is_azure_object_storage(chunk):
-    return urllib.parse.urlparse(chunk.url).netloc.endswith('core.windows.net')
-
-
 async def decrypt_object(file_id, encrypted_object, encryption_key, chunk):
     """
     Decrypt Encrypted Object.
@@ -83,7 +83,7 @@ async def decrypt_object(file_id, encrypted_object, encryption_key, chunk):
     :rtype: bytes
     """
     try:
-        return decrypt_block(encrypted_object[16:] if is_azure_object_storage(chunk) else encrypted_object, encryption_key)
+        return decrypt_block(encrypted_object, encryption_key)
     except DirectIOError:
         logger.error('Failed to decrypt block.')
         raise DecryptBlockError(file_id, chunk)
@@ -123,14 +123,17 @@ async def process_chunk(client, file_id, chunk, encryption_key, semaphore):
     :rtype: cterasdk.direct.types.Block
     """
     async def process(client, chunk, encryption_key):
-        message = f"Processing block (offset={chunk.offset}, length={chunk.length}) "
+        message = (
+            f"Processing block #{chunk.number} "
+            f"(offset={chunk.offset}, length={chunk.length})"
+        )
         if file_id:
             message = message + f" for file ID {file_id}"
         logger.debug(message)
         encrypted_object = await get_object(client, file_id, chunk)
         decrypted_object = await decrypt_object(file_id, encrypted_object, encryption_key, chunk)
         decompressed_object = await decompress_object(file_id, decrypted_object, chunk)
-        return Block(file_id, chunk.offset, decompressed_object, chunk.length)
+        return Block(file_id, chunk.number, chunk.offset, decompressed_object, chunk.length)
 
     if semaphore is not None:
         async with semaphore:
@@ -180,7 +183,7 @@ def decrypt_encryption_key(file_id, wrapped_key, secret_access_key):
 
 
 @execute_with_retries(retries=3, backoff=1, max_backoff=10)
-async def get_chunks(api, bearer, file_id, start=None, end=None):
+async def get_chunks(api, bearer, file_id):
     """
     Get Chunks.
 
@@ -192,15 +195,14 @@ async def get_chunks(api, bearer, file_id, start=None, end=None):
     """
     logger.debug('Listing blocks for file ID: %s', file_id)
     try:
-        params = {k: v for k, v in [('rangeStart', start), ('rangeEnd', end)] if v is not None}
-        response = await api.get(f'{file_id}', params=params, headers={'Authorization': bearer})
+        response = await api.get(f'{file_id}', headers={'Authorization': bearer})
         if not response.chunks:
             logger.error('Could not find blocks for file ID: %s.', file_id)
             raise BlocksNotFoundError(file_id)
         return Metadata(file_id, response)
     except BadRequest as error:
         raise ObjectNotFoundError(file_id) from error
-    except (Unauthorized, Forbidden) as error:
+    except Unauthorized as error:
         raise AuthorizationError(file_id) from error
     except Unprocessable as error:
         raise UnsupportedStorageError(file_id) from error

cterasdk/direct/types.py

@@ -56,177 +56,89 @@ class CompressionLib:
 
 class Chunk(Object):
 
-    def __init__(self, offset, url, length):
+    def __init__(self, number, offset, url, length):
         """
         Initialize a Chunk.
 
+        :param int number: Chunk number.
         :param int offset: Chunk offset.
         :param str url: Signed URL.
         :param int length: Object length.
         """
         super().__init__(
+            number=number,
             offset=offset,
             url=url,
             length=length
         )
 
 
-class MetadataPart(Object):
+class Metadata(Object):
     """
-    CTERA Direct I/O File Metadata Part
-
-    :ivar int start: Starting offset.
-    :ivar int end: Ending offset.
-    :ivar int length: Length of the range in bytes.
-    :ivar bool encrypted: Indicates whether the range data is encrypted.
-    :ivar str encryption_key: Encryption key used for the range data.
-    :ivar bool compressed: Indicates whether the range data is compressed.
-    :ivar str compression_alg: Compression algorithm used for the range data.
-    :ivar list[cterasdk.common.object.Object] chunks: List of Chunk Objects.
+    CTERA Direct IO File Metadata
     """
-    def __init__(self, offset, end, encrypted, encryption_key, compressed, compression_alg, chunks):
+
+    def __init__(self, file_id, server_object):
         """
         Initialize a Direct IO metadata response object.
 
-        :param int offset: Starting offset.
-        :param int end: Ending offset.
-        :param bool encrypted: Indicates whether the file is encrypted.
-        :param str encryption_key: Encryption key used for the file.
-        :param bool compressed: Indicates whether the file is compressed.
-        :param str compression_alg: Compression algorithm used for the file.
-        :param list[cterasdk.common.object.Object] chunks: List of Chunk Objects.
+        :param int file_id: File ID.
+        :param cterasdk.common.object.Object server_object: Response Object.
         """
         super().__init__(
-            start=offset,
-            end=end,
-            length=(end - offset) + 1,
-            encrypted=encrypted,
-            encryption_key=encryption_key,
-            compressed=compressed,
-            compression_alg=compression_alg
+            file_id=file_id,
+            encrypted=server_object.encrypt_info.data_encrypted,
+            compressed=server_object.compression_type != CompressionLib.Off,
+            chunks=Metadata._format_chunks(server_object.chunks)
         )
-        for chunk in chunks:
-            self.chunks.append(Chunk(offset, chunk.url, chunk.len))
-            offset = offset + chunk.len
+        self.encryption_key = server_object.encrypt_info.wrapped_key if self.encrypted else None
+        self.compression_library = server_object.compression_type if self.compressed else None
+        last_chunk = self.chunks[-1]
+        self.size = last_chunk.offset + last_chunk.length
 
     @staticmethod
-    def from_server_object(server_object):
-        compressed = server_object.compression_type != CompressionLib.Off
-        start, end = map(int, server_object.actual_blocks_range.range.split('-'))
-        return MetadataPart(
-            start,
-            end,
-            server_object.encrypt_info.data_encrypted,
-            server_object.encrypt_info.wrapped_key if server_object.encrypt_info.data_encrypted else None,
-            compressed,
-            server_object.compression_type if compressed else None
-        )
-
-    def __repr__(self):
-        return str(self)
-
-    def __str__(self):
-        return (
-            f"{self.__class__.__name__}("
-            f"{{'start': {self.start}, "
-            f"'end': {self.end}, "
-            f"'encrypted': {self.encrypted}, "
-            f"'compressed': {self.compressed}, "
-            f"'length': {self.length}, "
-            f"'chunks': {len(self.chunks)}}})"
-        )
-
-
-class Metadata(Object):
-    """
-    CTERA Direct I/O File Metadata
-
-    :ivar int file_id: File ID.
-    :ivar int file_size: File Size.
-    :ivar list[cterasdk.direct.types.MetadataPart] parts: List of Metadata Parts.
-    :ivar bool encrypted: Indicates whether the range data is encrypted.
-    :ivar str encryption_key: Encryption key used for the range data.
-    :ivar bool compressed: Indicates whether the range data is compressed.
-    :ivar str compression_alg: Compression algorithm used for the range data.
-    """
-    def __init__(self, file_id, *parts):
+    def _format_chunks(server_object):
         """
-        Initialize a Direct IO metadata response object.
+        Create Chunks.
 
         :param int file_id: File ID.
-        :param list[cterasdk.direct.types.MetadataPart] parts: List of Metadata Parts.
+        :param cterasdk.common.object.Object server_object: Server response.
+        :param list[int] blocks: List of block numbers to retrieve.
+        :returns: Chunk objects
+        :rtype: list[cterasdk.direct.types.Chunk]
         """
-        super().__init__(file_id=file_id, parts=parts)
-
-    @property
-    def size(self):
-        return self.parts[0].actual_blocks_range.file_size
-
-    @property
-    def encrypted(self):
-        return self.parts[0].encrypted
-
-    @property
-    def encryption_key(self):
-        return self.parts[0].encryption_key
-
-    @property
-    def compressed(self):
-        return self.parts[0].compressed
-
-    @property
-    def compression_alg(self):
-        return self.parts[0].compression_alg
-
-    @property
-    def start(self):
-        return self.parts[0].start
-
-    @property
-    def end(self):
-        return self.parts[-1].end
+        offset = 0
+        chunks = []
+        for number, chunk in enumerate(server_object, 1):
+            chunks.append(Chunk(number, offset, chunk.url, chunk.len))
+            offset = offset + chunk.len
+        return chunks
 
     def serialize(self):
         """
-        Serialize Direct I/O metadata to a dictionary.
+        Serialize Direct IO metadata to a dictionary.
         """
-        chunks = []
-        for part in self.parts:
-            chunks.extend(part.chunks)
-
-        return {
-            'file_id': self.file_id,
-            'size': self.size,
-            'encrypted': self.encrypted,
-            'encryption_key': utils.utf8_decode(base64.b64encode(self.encryption_key)) if self.encrypted else None,
-            'compressed': self.compressed,
-            'compression_alg': self.compression_alg,
-            'chunks': chunks
-        }
-
-    def __repr__(self):
-        return str(self)
-
-    def __str__(self):
-        d = self.serialize()
-        d.pop('encryption_key')
-        d['chunks'] = len(d['chunks'])
-        return d
+        x = copy.deepcopy(self)
+        if self.encrypted:
+            x.encryption_key = utils.utf8_decode(base64.b64encode(self.encryption_key))
+        return x
 
 
 class Block:
     """Block"""
 
-    def __init__(self, file_id, offset, data, length):
+    def __init__(self, file_id, number, offset, data, length):
         """
         Initialize a Block.
 
         :param int file_id: File ID.
+        :param int number: Block number.
         :param int offset: Block offset.
         :param bytes data: Bytes
         :param int length: Block length.
         """
         self._file_id = file_id
+        self._number = number
         self._offset = offset
         self._data = data
         self._length = length
@@ -235,6 +147,10 @@ def __init__(self, file_id, offset, data, length):
     def file_id(self):
         return self._file_id
 
+    @property
+    def number(self):
+        return self._number
+
     @property
     def offset(self):
         return self._offset