Added robust checks, and changed the api layout again. Byte offset mode parameter was moved to ZipFly instead of stream()

Author: Pamparampam

MANIFEST.IN

@@ -1,2 +1,3 @@
 exclude .gitignore
-exclude dev-requirements.txt
+exclude dev-requirements.txt
+prune tests

README.md

@@ -76,7 +76,7 @@ file2 = LocalFile(file_path='files/as61aade2ebfd.mp4', compression_method=consts
 files = [file1, file2]
 
 zipFly = ZipFly(files)
-archive_size = zipFly.calculate_archive_size() # raises ValueError if it can't calculate size
+archive_size = zipFly.calculate_archive_size() # raises RuntimeError if it can't calculate size
 
 # for example you can set as content length in http response
 response['Content-Length'] = archive_size
@@ -143,11 +143,11 @@ async def async_save_pause():
 file3 = GenFile(name="file.txt", generator=file_generator(), crc=crc)
 file4 = LocalFile(file_path='public/2ae9dcd01a3aa.mp4', name="files/my_file2.mp4")
 files2 = [file3, file4]
-zipFly2 = ZipFly(files2)
+resumeZipFly = ZipFly(files2, byte_offset=STOP_BYTE)
 
 async def async_save_resume():
     with open("out/file.zip", 'ab') as f_out: # Append mode
-        async for chunk in zipFly2.async_stream(byte_offset=STOP_BYTE):
+        async for chunk in resumeZipFly.async_stream():
             f_out.write(chunk)
 
 async def pause_resume_save():

src/zipFly/BaseFile.py

@@ -7,29 +7,34 @@
 
 
 class BaseFile(ABC):
-    def __init__(self, compression_method: int):
+    def __init__(self, name: str, compression_method: int):
         self.__used = False
         self.__compressed_size = 0
         self.__offset = 0  # Offset to local file header
         self.__crc = 0
         self.__compression_method = compression_method or consts.NO_COMPRESSION
         self.__flags = 0b00001000  # flag about using data descriptor is always on
+        self.__byte_offset_mode = False
+        if name == "":
+            raise KeyError("File name cannot be blank.")
+        self._name = name
 
     def __str__(self):
-        return f"FILE[{self.name}]"
+        return f"BaseFile[name={self.name}]"
+
+    def __repr__(self):
+        return f"BaseFile({self.name})"
 
     def _check_if_used(self):
         if self.__used:
             raise RuntimeError("Do not re-use file instances. Recreate it.")
         self.__used = True
 
     def generate_processed_file_data(self) -> Generator[bytes, None, None]:
+        """Generates compressed file data"""
         self._check_if_used()
         compressor = Compressor(self)
 
-        """
-        Generates compressed file data
-        """
         for chunk in self._generate_file_data():
             chunk = compressor.process(chunk)
             if len(chunk) > 0:
@@ -39,12 +44,10 @@ def generate_processed_file_data(self) -> Generator[bytes, None, None]:
             yield chunk
 
     async def async_generate_processed_file_data(self) -> AsyncGenerator[bytes, None]:
+        """Generates compressed file data"""
         self._check_if_used()
         compressor = Compressor(self)
 
-        """
-        Generates compressed file data
-        """
         async for chunk in self._async_generate_file_data():
             chunk = compressor.process(chunk)
             if len(chunk) > 0:
@@ -85,6 +88,15 @@ def get_crc(self) -> int:
     def set_crc(self, new_crc) -> None:
         self.__crc = new_crc
 
+    def set_byte_offset_mode(self, value) -> None:
+        self.__byte_offset_mode = value
+
+    def is_byte_offset_mode(self) -> bool:
+        return self.__byte_offset_mode
+
+    def set_file_name(self, new_name: str) -> None:
+        self._name = new_name
+
     @property
     def file_path_bytes(self) -> bytes:
         try:
@@ -102,11 +114,7 @@ async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
         raise NotImplementedError
 
     @abstractmethod
-    def set_file_name(self, new_name: str) -> None:
-        raise NotImplementedError
-
-    @abstractmethod
-    def calculate_crc(self) -> int:
+    def get_predicted_crc(self) -> int:
         raise NotImplementedError
 
     @property
@@ -117,14 +125,14 @@ def size(self) -> int:
     def modification_time(self) -> float:
         raise NotImplementedError
 
-    @property
-    def name(self) -> str:
-        raise NotImplementedError
-
     @property
     def flags(self) -> int:
         return self.__flags
 
+    @property
+    def name(self) -> str:
+        return self._name
+
     @property
     def compression_method(self) -> int:
         return self.__compression_method

src/zipFly/Compressor.py

@@ -12,6 +12,8 @@ def __init__(self, file: 'BaseFile'):
             self.compr = zlib.compressobj(5, zlib.DEFLATED, -15)
             self.process = self._process_deflate
             self.tail = self._tail_deflate
+        else:
+            raise KeyError("Unknown compression method in compressor")
 
     # no compression
     def _process_through(self, chunk):

src/zipFly/EmptyFolder.py

@@ -0,0 +1,38 @@
+import time
+from typing import Generator, AsyncGenerator
+
+from . import consts
+from .BaseFile import BaseFile
+
+
+class EmptyFolder(BaseFile):
+    def __init__(self, name: str, modification_time: float = None):
+        super().__init__(name, consts.NO_COMPRESSION)
+        if not name.endswith("/"):
+            name += "/"
+        self._name = name
+        self._modification_time = modification_time if modification_time else time.time()
+        self._finished_streaming = False
+
+    def __str__(self):
+        return f"EmptyFolder[name={self.name}]"
+
+    def __repr__(self):
+        return f"EmptyFolder({self.name})"
+
+    def _generate_file_data(self) -> Generator[bytes, None, None]:
+        yield b''
+
+    async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
+        yield b''
+
+    @property
+    def size(self) -> int:
+        return 0
+
+    @property
+    def modification_time(self) -> float:
+        return self._modification_time
+
+    def get_predicted_crc(self) -> int:
+        return 0

src/zipFly/GenFile.py

@@ -8,49 +8,72 @@
 class GenFile(BaseFile):
     """DO NOT REUSE GenFile instances!"""
     def __init__(self, name: str, generator: Union[Generator[bytes, None, None], AsyncGenerator[bytes, None]], compression_method: int = consts.NO_COMPRESSION, modification_time: float = None, size: int = None, crc: int = None):
-        super().__init__(compression_method)
-        self._name = name
+        if size and compression_method != consts.NO_COMPRESSION:
+            raise ValueError("File size is allowed only with NO_COMPRESSION")
+
+        super().__init__(name, compression_method)
         self._generator = generator
-        self._size = size
-        self._overriden_crc = crc  # used in byte offset mode
+        self._predicted_size = size
+        self._streamed_size = 0
+        self._predicted_crc = crc  # used in byte offset mode
         self._modification_time = modification_time if modification_time else time.time()
+        self._finished_streaming = False
+
+    def __str__(self):
+        return f"GenFile[name={self.name}]"
+
+    def __repr__(self):
+        return f"GenFile({self.name})"
 
     def _get_generator(self):
         return self._generator
 
     def _generate_file_data(self) -> Generator[bytes, None, None]:
         generator = self._get_generator()
         if isinstance(generator, Generator):
-            yield from generator
+            for chunk in generator:
+                self._streamed_size += len(chunk)
+                yield chunk
         else:
             raise ValueError(f"generator must be of type Generator, not '{type(generator)}'")
 
+        self._finish_and_validate()
+
     async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
         generator = self._get_generator()
         if isinstance(generator, AsyncGenerator):
             async for chunk in generator:
+                self._streamed_size += len(chunk)
                 yield chunk
         else:
             raise ValueError(f"generator must be of type AsyncGenerator, not '{type(generator)}'")
 
-    @property
-    def name(self) -> str:
-        return self._name
+        self._finish_and_validate()
+
+    def _finish_and_validate(self):
+        self._finished_streaming = True
+        if self._predicted_size is not None and self._predicted_size != self._streamed_size:
+            raise RuntimeError(f"Size({self._predicted_size}) != streamed size({self._streamed_size})")
+
+        if self._predicted_crc is not None and self._predicted_crc != self.get_crc():
+            raise RuntimeError(f"Crc({self._predicted_crc}) != streamed crc({self.get_crc()})")
 
     @property
     def size(self) -> int:
-        if self._size is not None:
-            return self._size
-        raise RuntimeError("Archive size not known before streaming. Probably GenFile() is missing size attribute.")
+        if not self._finished_streaming:
+            if self.is_byte_offset_mode():
+                if self._predicted_size is None:
+                    raise RuntimeError("Archive size not known before streaming. Probably GenFile() is missing size attribute.")
+                return self._predicted_size
+        return self._streamed_size
 
     @property
     def modification_time(self) -> float:
         return self._modification_time
 
-    def set_file_name(self, new_name: str) -> None:
-        self._name = new_name
-
-    def calculate_crc(self) -> int:
-        if self._overriden_crc:
-            return self._overriden_crc
-        raise ValueError("Crc must be explicitly set to allow for byte offset streaming!")
+    def get_predicted_crc(self) -> int:
+        if self.is_byte_offset_mode():
+            if self._predicted_crc is None:
+                raise RuntimeError("Crc not known before streaming. Probably GenFile() is missing crc attribute.")
+            return self._predicted_crc
+        raise ValueError("Attempted to get predicted crc without byte offset mode")

src/zipFly/LocalFile.py

@@ -17,12 +17,19 @@ def __init__(self, file_path: Union[str, Path], name: str = None, compression_me
 
         self._file_path = str(file_path)
         self.chunk_size = chunk_size
-        self._name = name if name else self._file_path
-        super().__init__(compression_method)
+        name = name if name else self._file_path
+        super().__init__(name, compression_method)
+
+    def __str__(self):
+        return f"LocalFile[name={self.name}]"
+
+    def __repr__(self):
+        return f"LocalFile({self.name})"
 
     async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
         if not self.chunk_size:
             self.chunk_size = 1048 * 1048 * 4
+
         async with aiofiles.open(self._file_path, "rb") as fh:
             while True:
                 part = await fh.read(self.chunk_size)
@@ -41,10 +48,6 @@ def _generate_file_data(self) -> Generator[bytes, None, None]:
                     break
                 yield chunk
 
-    @property
-    def name(self) -> str:
-        return self._name
-
     @property
     def size(self) -> int:
         return os.path.getsize(self._file_path)
@@ -54,10 +57,7 @@ def modification_time(self) -> float:
         """Returns the modification time as a Unix timestamp"""
         return os.path.getmtime(self._file_path)
 
-    def set_file_name(self, new_name: str) -> None:
-        self._name = new_name
-
-    def calculate_crc(self) -> int:
+    def get_predicted_crc(self) -> int:
         crc = 0
         with open(self._file_path, "rb") as f:
             while chunk := f.read(self.chunk_size):